From 049118f50ce46bf837de7bbb55486e74a9dc9e5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 16 Dec 2021 13:30:09 -0600 Subject: [PATCH] ARROW-15133: [CI] Remove util_checkout.sh and util_cleanup.sh scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `ci/scripts/util_checkout.sh` was used to checkout submodules because `actions/checkout@v2` has removed support for that, but they have restored it since. - `ci/scripts/util_cleanup.sh` was used to free up disk space on github actions runners, because at that time it was limited to 7GB, from a recent run it looks like the linux runners now have 32GB free space so we can try to disable the cleanup step sparing almost a minute of build time Closes #11974 from kszucs/checkout Authored-by: Krisztián Szűcs Signed-off-by: Jonathan Keane --- .github/workflows/archery.yml | 2 - .github/workflows/cpp.yml | 24 +++------ .github/workflows/csharp.yml | 9 ---- .github/workflows/dev.yml | 10 +--- .github/workflows/docs.yml | 4 -- .github/workflows/go.yml | 26 +-------- .github/workflows/integration.yml | 5 +- .github/workflows/java.yml | 11 +--- .github/workflows/java_jni.yml | 10 +--- .github/workflows/js.yml | 9 +--- .github/workflows/matlab.yml | 2 - .github/workflows/python.yml | 9 +--- .github/workflows/r-without-arrow.yml | 4 -- .github/workflows/r.yml | 8 --- .github/workflows/ruby.yml | 15 ++---- ci/scripts/util_checkout.sh | 30 ----------- ci/scripts/util_cleanup.sh | 54 ------------------- dev/archery/archery/crossbow/core.py | 24 +++++---- dev/tasks/cpp-examples/github.linux.yml | 21 ++------ dev/tasks/docker-tests/github.linux.yml | 4 -- dev/tasks/fuzz-tests/github.oss-fuzz.yml | 4 -- .../linux-packages/github.linux.amd64.yml | 3 -- dev/tasks/macros.jinja | 22 ++++---- ...github.linux.arrow.version.back.compat.yml | 29 +++------- dev/tasks/r/github.linux.cran.yml | 32 +++-------- dev/tasks/r/github.linux.offline.build.yml | 31 +++-------- dev/tasks/r/github.linux.rchk.yml | 20 ++----- dev/tasks/r/github.linux.revdepcheck.yml | 32 +++-------- dev/tasks/r/github.linux.versions.yml | 32 +++-------- dev/tasks/r/github.macos-linux.local.yml | 17 ++---- dev/tasks/r/github.macos.autobrew.yml | 17 ++---- 31 files changed, 89 insertions(+), 431 deletions(-) delete mode 100755 ci/scripts/util_checkout.sh delete mode 100755 ci/scripts/util_cleanup.sh diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml index 78aa9f4b7ba00..b7321a7f65d6f 100644 --- a/.github/workflows/archery.yml +++ b/.github/workflows/archery.yml @@ -50,8 +50,6 @@ jobs: - name: Git Fixup shell: bash run: git branch master origin/master || true - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Setup Python uses: actions/setup-python@v1 with: diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 468b3ea86870d..b2465313a166d 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -70,10 +70,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh + submodules: recursive - name: Cache Docker Volumes uses: actions/cache@v2 with: @@ -108,6 +105,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 + submodules: recursive - name: Check CMake presets run: | cd cpp @@ -148,9 +146,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh + submodules: recursive - name: Install Dependencies shell: bash run: | @@ -241,13 +237,10 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh + submodules: recursive - name: Build shell: bash - run: | - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build + run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - name: Test shell: bash run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build @@ -305,17 +298,14 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh + submodules: recursive - uses: msys2/setup-msys2@v2 with: msystem: MINGW${{ matrix.mingw-n-bits }} update: true - name: Setup MSYS2 shell: msys2 {0} - run: | - ci/scripts/msys2_setup.sh cpp + run: ci/scripts/msys2_setup.sh cpp - name: Cache ccache uses: actions/cache@v2 with: diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 59e5113685fc9..173c2fb2b2fcc 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -53,9 +53,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - name: Install Source Link shell: bash run: dotnet tool install --global sourcelink @@ -84,9 +81,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - name: Install Source Link run: dotnet tool install --global sourcelink - name: Build @@ -117,9 +111,6 @@ jobs: - name: Install Source Link shell: bash run: dotnet tool install --global sourcelink - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - name: Build shell: bash run: ci/scripts/csharp_build.sh $(pwd) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index d380f9076b019..e4544611b2ab5 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -41,10 +41,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Setup Python uses: actions/setup-python@v1 with: @@ -75,9 +71,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - name: Install Python uses: actions/setup-python@v1 with: @@ -96,5 +89,4 @@ jobs: ci/scripts/release_test.sh $(pwd) - name: Run Merge Script Test shell: bash - run: | - pytest -v dev/test_merge_arrow_pr.py + run: pytest -v dev/test_merge_arrow_pr.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 19bebedb46e50..d7673c2030214 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -53,10 +53,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Cache Docker Volumes uses: actions/cache@v2 with: diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1c92bb3a03117..e986471b832cb 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -58,10 +58,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Setup Python uses: actions/setup-python@v1 with: @@ -91,10 +87,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Setup Python uses: actions/setup-python@v1 with: @@ -125,10 +117,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Setup Python uses: actions/setup-python@v1 with: @@ -160,9 +148,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - name: Build shell: bash run: ci/scripts/go_build.sh . @@ -188,9 +173,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - name: Build shell: bash run: ci/scripts/go_build.sh . @@ -218,9 +200,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - name: Brew Install Arrow shell: bash run: brew install apache-arrow @@ -257,9 +236,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - uses: msys2/setup-msys2@v2 with: msystem: MINGW${{ matrix.mingw-n-bits }} @@ -273,7 +249,7 @@ jobs: run: | echo "CGO_CPPFLAGS=-I$(cygpath --windows ${MINGW_PREFIX}/include)" >> $GITHUB_ENV echo "CGO_LDFLAGS=-g -O2 -L$(cygpath --windows ${MINGW_PREFIX}/lib) -L$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_ENV - echo "GOROOT=$(cygpath --windows ${MINGW_PREFIX}/lib/go)" >> $GITHUB_ENV + echo "GOROOT=$(cygpath --windows ${MINGW_PREFIX}/lib/go)" >> $GITHUB_ENV echo "GOPATH=$(cygpath --windows ${HOME}/gopath)" >> $GITHUB_ENV mkdir -p $(cygpath --windows ${HOME}/gopath) echo "MINGW_PREFIX=$(cygpath --windows ${MINGW_PREFIX})" >> $GITHUB_ENV diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c93d4133fc7cc..1ee7c695de9b1 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -62,15 +62,12 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh + submodules: recursive - name: Checkout Arrow Rust uses: actions/checkout@v2 with: repository: apache/arrow-rs path: rust - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Cache Docker Volumes uses: actions/cache@v2 with: diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index df942771bba25..58d669e94b024 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -64,12 +64,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh + submodules: recursive - name: Cache Docker Volumes uses: actions/cache@v2 with: @@ -107,9 +102,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh + submodules: recursive - name: Build shell: bash run: ci/scripts/java_build.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index d4296b674dfb4..807289c8a999b 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -56,10 +56,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh + submodules: recursive - name: Cache Docker Volumes uses: actions/cache@v2 with: @@ -89,10 +86,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh + submodules: recursive - name: Cache Docker Volumes uses: actions/cache@v2 with: diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index 8f8c43403ee4b..e3a56b0b39fa0 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -51,10 +51,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Setup Python uses: actions/setup-python@v1 with: @@ -85,9 +81,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - name: Install NodeJS uses: actions/setup-node@v1 with: @@ -111,7 +104,7 @@ jobs: - name: Checkout Arrow uses: actions/checkout@v1 with: - submodules: true + fetch-depth: 0 - name: Install NodeJS uses: actions/setup-node@v1 with: diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml index 268bb63cb2f2c..b403fbc3366ae 100644 --- a/.github/workflows/matlab.yml +++ b/.github/workflows/matlab.yml @@ -45,8 +45,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - name: Install ninja-build run: sudo apt-get install ninja-build - name: Install MATLAB diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3a77e9e5dceed..7a220d61efd68 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -89,10 +89,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh + submodules: recursive - name: Cache Docker Volumes uses: actions/cache@v2 with: @@ -143,9 +140,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh + submodules: recursive - name: Install Dependencies shell: bash run: | diff --git a/.github/workflows/r-without-arrow.yml b/.github/workflows/r-without-arrow.yml index 12afdd43c4aeb..271351eac2c6f 100644 --- a/.github/workflows/r-without-arrow.yml +++ b/.github/workflows/r-without-arrow.yml @@ -53,10 +53,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Cache Docker Volumes uses: actions/cache@v2 with: diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 4f8709fe5bc4a..ef857eba160e7 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -67,10 +67,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Cache Docker Volumes uses: actions/cache@v2 with: @@ -129,10 +125,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - run: ci/scripts/util_cleanup.sh - name: Cache Docker Volumes uses: actions/cache@v2 with: diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 5e861ecf2cfd7..d0a4c4f433392 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -71,12 +71,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: ci/scripts/util_cleanup.sh + submodules: recursive - name: Cache Docker Volumes uses: actions/cache@v2 with: @@ -132,9 +127,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh + submodules: recursive - name: Install Homebrew Dependencies shell: bash run: | @@ -228,9 +221,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh + submodules: recursive - name: Setup Ruby uses: ruby/setup-ruby@v1 with: diff --git a/ci/scripts/util_checkout.sh b/ci/scripts/util_checkout.sh deleted file mode 100755 index 107f2f0e9b2bd..0000000000000 --- a/ci/scripts/util_checkout.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# this script is github actions specific to check out the submodules and tags - -set -ex - -# TODO(kszucs): remove it once the "submodules: recursive" feature is released -auth_header="$(git config --local --get http.https://github.com/.extraheader)" -git submodule sync --recursive -git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 - -# fetch all the tags -git fetch --depth=1 origin +refs/tags/*:refs/tags/* diff --git a/ci/scripts/util_cleanup.sh b/ci/scripts/util_cleanup.sh deleted file mode 100755 index 3a13a1a784afd..0000000000000 --- a/ci/scripts/util_cleanup.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This script is Github Actions-specific to free up disk space, -# to avoid disk full errors on some builds - -if [ $RUNNER_OS = "Linux" ]; then - df -h - - # remove swap - sudo swapoff -a - sudo rm -f /swapfile - - # clean apt cache - sudo apt clean - - # remove haskell, consumes 8.6 GB - sudo rm -rf /opt/ghc - - # 1 GB - sudo rm -rf /home/linuxbrew/.linuxbrew - - # 1+ GB - sudo rm -rf /opt/hostedtoolcache/CodeQL - - # 1+ GB - sudo rm -rf /usr/share/swift - - # 12 GB, but takes a lot of time to delete - #sudo rm -rf /usr/local/lib/android - - # remove cached docker images, around 13 GB - docker rmi $(docker image ls -aq) - - # NOTE: /usr/share/dotnet is 25 GB -fi - -df -h diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 06a0b664f2085..bef890b978ee5 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -205,6 +205,17 @@ def _git_ssh_to_https(url): return url.replace('git@github.com:', 'https://github.com/') +def _parse_github_user_repo(remote_url): + m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', remote_url) + if m is None: + raise CrossbowError( + "Unable to parse the github owner and repository from the " + "repository's remote url '{}'".format(remote_url) + ) + user, repo = m.group(1), m.group(2) + return user, repo + + class Repo: """ Base class for interaction with local git repositories @@ -388,23 +399,13 @@ def file_contents(self, commit_id, file): blob = self.repo[entry.id] return blob.data - def _parse_github_user_repo(self): - m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url) - if m is None: - raise CrossbowError( - "Unable to parse the github owner and repository from the " - "repository's remote url '{}'".format(self.remote_url) - ) - user, repo = m.group(1), m.group(2) - return user, repo - def as_github_repo(self, github_token=None): """Converts it to a repository object which wraps the GitHub API""" if self._github_repo is None: if not _have_github3: raise ImportError('Must install github3.py') github_token = github_token or self.github_token - username, reponame = self._parse_github_user_repo() + username, reponame = _parse_github_user_repo(self.remote_url) session = github3.session.GitHubSession( default_connect_timeout=10, default_read_timeout=30 @@ -679,6 +680,7 @@ def __init__(self, head, branch, remote, version, email=None): self.email = email self.branch = branch self.remote = remote + self.github_repo = "/".join(_parse_github_user_repo(remote)) self.version = version self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) # Semantic Versioning 1.0.0: https://semver.org/spec/v1.0.0.html diff --git a/dev/tasks/cpp-examples/github.linux.yml b/dev/tasks/cpp-examples/github.linux.yml index 717d3c44302a5..fe26eb1e24e0e 100644 --- a/dev/tasks/cpp-examples/github.linux.yml +++ b/dev/tasks/cpp-examples/github.linux.yml @@ -15,30 +15,17 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: test: name: C++ Example runs-on: ubuntu-latest steps: - - name: Checkout Arrow - shell: bash - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh + {{ macros.github_checkout_arrow()|indent }} + - name: Run shell: bash run: | diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index b3c9b55cb5125..0ae4664a5e972 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -34,10 +34,6 @@ jobs: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_install_archery()|indent }} - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Execute Docker Build shell: bash run: | diff --git a/dev/tasks/fuzz-tests/github.oss-fuzz.yml b/dev/tasks/fuzz-tests/github.oss-fuzz.yml index 43958b51ac100..e49d73d8d11e7 100644 --- a/dev/tasks/fuzz-tests/github.oss-fuzz.yml +++ b/dev/tasks/fuzz-tests/github.oss-fuzz.yml @@ -38,10 +38,6 @@ jobs: run: | git clone --depth=50 https://github.com/google/oss-fuzz.git - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Install dependencies working-directory: oss-fuzz run: | diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml index 557c4ab41305b..496ca49213f30 100644 --- a/dev/tasks/linux-packages/github.linux.amd64.yml +++ b/dev/tasks/linux-packages/github.linux.amd64.yml @@ -31,9 +31,6 @@ jobs: uses: ruby/setup-ruby@v1 with: ruby-version: '3.0' - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - name: Cache ccache uses: actions/cache@v2 with: diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index fb80d3b2f3145..d8f2816eeea71 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -27,12 +27,12 @@ on: {%- macro github_checkout_arrow() -%} - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow config core.symlinks true - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive + uses: actions/checkout@v2 + with: + repository: {{ arrow.github_repo }} + ref: {{ arrow.head }} + path: arrow + submodules: recursive {% endmacro %} {%- macro github_login_dockerhub() -%} @@ -102,9 +102,8 @@ on: {%- macro azure_checkout_arrow() -%} - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD + git clone --no-checkout --branch {{ arrow.branch }} {{ arrow.remote }} arrow + git -C arrow checkout {{ arrow.head }} git -C arrow submodule update --init --recursive displayName: Clone arrow {% endmacro %} @@ -149,9 +148,8 @@ on: {% endmacro %} {%- macro travis_checkout_arrow() -%} - - git clone --no-checkout {{ arrow.remote }} arrow - - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - - git -C arrow checkout FETCH_HEAD + - git clone --no-checkout --branch {{ arrow.branch }} {{ arrow.remote }} arrow + - git -C arrow checkout {{ arrow.head }} - git -C arrow submodule update --init --recursive {% endmacro %} diff --git a/dev/tasks/r/github.linux.arrow.version.back.compat.yml b/dev/tasks/r/github.linux.arrow.version.back.compat.yml index e48b67ac643ac..c743454887cbb 100644 --- a/dev/tasks/r/github.linux.arrow.version.back.compat.yml +++ b/dev/tasks/r/github.linux.arrow.version.back.compat.yml @@ -15,12 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push +{{ macros.github_header() }} jobs: write-files: @@ -32,18 +29,8 @@ jobs: ARROW_R_DEV: "TRUE" RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh + {{ macros.github_checkout_arrow()|indent }} + - uses: r-lib/actions/setup-r@v1 - name: Install dependencies run: | @@ -87,12 +74,8 @@ jobs: RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" OLD_ARROW_VERSION: {{ '${{ matrix.config.old_arrow_version }}' }} steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive + {{ macros.github_checkout_arrow()|indent }} + - uses: r-lib/actions/setup-r@v1 with: r-version: {{ '${{ matrix.config.r }}' }} diff --git a/dev/tasks/r/github.linux.cran.yml b/dev/tasks/r/github.linux.cran.yml index 03d22dcbfef1f..95866264ceedb 100644 --- a/dev/tasks/r/github.linux.cran.yml +++ b/dev/tasks/r/github.linux.cran.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: as-cran: @@ -44,27 +39,12 @@ jobs: R_TAG: "latest" ARROW_R_DEV: "FALSE" steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh - - name: Docker Pull - shell: bash - run: cd arrow && docker-compose pull --ignore-pull-failures r - - name: Docker Build - shell: bash - run: cd arrow && docker-compose build r + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + - name: Docker Run shell: bash - run: cd arrow && docker-compose run r + run: archery docker run r - name: Dump install logs run: cat arrow/r/check/arrow.Rcheck/00install.out if: always() diff --git a/dev/tasks/r/github.linux.offline.build.yml b/dev/tasks/r/github.linux.offline.build.yml index 60685b18c5cbc..cea1772b9654a 100644 --- a/dev/tasks/r/github.linux.offline.build.yml +++ b/dev/tasks/r/github.linux.offline.build.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: grab-dependencies: @@ -34,18 +29,8 @@ jobs: ARROW_R_DEV: "TRUE" RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh + {{ macros.github_checkout_arrow()|indent }} + - uses: r-lib/actions/setup-r@v1 - name: Pull Arrow dependencies run: | @@ -76,12 +61,8 @@ jobs: ARROW_R_DEV: TRUE RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive + {{ macros.github_checkout_arrow()|indent }} + - uses: r-lib/actions/setup-r@v1 - name: Download artifacts uses: actions/download-artifact@v2 diff --git a/dev/tasks/r/github.linux.rchk.yml b/dev/tasks/r/github.linux.rchk.yml index 72ff269693f37..9854e885f7ab6 100644 --- a/dev/tasks/r/github.linux.rchk.yml +++ b/dev/tasks/r/github.linux.rchk.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: as-cran: @@ -34,15 +29,8 @@ jobs: ARROW_R_DEV: "FALSE" RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh + {{ macros.github_checkout_arrow()|indent }} + - uses: r-lib/actions/setup-r@v1 - uses: r-lib/actions/setup-pandoc@v1 - name: Install dependencies diff --git a/dev/tasks/r/github.linux.revdepcheck.yml b/dev/tasks/r/github.linux.revdepcheck.yml index 80071171b7530..7f37fc1080224 100644 --- a/dev/tasks/r/github.linux.revdepcheck.yml +++ b/dev/tasks/r/github.linux.revdepcheck.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: r-versions: @@ -36,27 +31,12 @@ jobs: R_TAG: "latest-focal" ARROW_R_DEV: "TRUE" steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh - - name: Docker Pull - shell: bash - run: cd arrow && docker-compose pull --ignore-pull-failures r - - name: Docker Build - shell: bash - run: cd arrow && docker-compose build r-revdepcheck + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + - name: Docker Run shell: bash - run: cd arrow && docker-compose run r-revdepcheck + run: archery docker run r-revdepcheck - name: revdepcheck CRAN report if: always() shell: bash diff --git a/dev/tasks/r/github.linux.versions.yml b/dev/tasks/r/github.linux.versions.yml index f383fe8d07f65..0e6178ab8b380 100644 --- a/dev/tasks/r/github.linux.versions.yml +++ b/dev/tasks/r/github.linux.versions.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: r-versions: @@ -46,27 +41,12 @@ jobs: R_TAG: "{{ MATRIX }}-bionic" ARROW_R_DEV: "TRUE" steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh - - name: Docker Pull - shell: bash - run: cd arrow && docker-compose pull --ignore-pull-failures r - - name: Docker Build - shell: bash - run: cd arrow && docker-compose build r + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + - name: Docker Run shell: bash - run: cd arrow && docker-compose run r + run: archery docker run r - name: Dump install logs run: cat arrow/r/check/arrow.Rcheck/00install.out if: always() diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml index 79e3332af4bc0..e2363e0b96710 100644 --- a/dev/tasks/r/github.macos-linux.local.yml +++ b/dev/tasks/r/github.macos-linux.local.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: autobrew: @@ -34,12 +29,8 @@ jobs: os: [macOS-latest, ubuntu-20.04] steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive + {{ macros.github_checkout_arrow()|indent }} + - name: Configure non-autobrew dependencies (macos) run: | cd arrow/r diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml index 1b8500f64b319..3a45bad0110fa 100644 --- a/dev/tasks/r/github.macos.autobrew.yml +++ b/dev/tasks/r/github.macos.autobrew.yml @@ -15,26 +15,17 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: autobrew: name: "Autobrew" runs-on: macOS-latest steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive + {{ macros.github_checkout_arrow()|indent }} + - name: Configure autobrew script run: | cd arrow/r