From 9fb67abe619baa6b1105c5e46b825e8218d518cb Mon Sep 17 00:00:00 2001 From: Nicola Sella Date: Thu, 7 Sep 2023 15:15:00 +0200 Subject: [PATCH 1/5] Add ccache Github workflow This workflow aims to replace the current ci action[1] exploiting ccache to run cmake builds quicker. It shares cache for the same PRs and for the branch each PR is opened against. [1]: https://github.com/rpm-software-management/dnf5/blob/main/.github/workflows/ci.yml#L9-L45 --- .github/workflows/ccache-build.yml | 75 ++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/ccache-build.yml diff --git a/.github/workflows/ccache-build.yml b/.github/workflows/ccache-build.yml new file mode 100644 index 000000000..4bc99090b --- /dev/null +++ b/.github/workflows/ccache-build.yml @@ -0,0 +1,75 @@ +--- +name: CCache Build +on: + pull_request: + workflow_dispatch: + +jobs: + build-dnf5: + name: CMake Build + runs-on: ubuntu-latest + container: + image: fedora:rawhide + strategy: + fail-fast: false # don't fail all matrix jobs if one of them fails + matrix: + compiler: ['gcc', clang] # gcc is the default + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # this step needs to be incorporated in our container that we use for testing + - name: Install dependencies + run: | + dnf install -y ccache clang && dnf builddep -y ./dnf5.spec + + - name: Configure CC CXX env variables + id: configure_compiler + run: | + if [[ "${{matrix.compiler}}" -eq "clang" ]]; then + echo "cxx=/usr/lib64/ccache/clang++" >> "$GITHUB_OUTPUT" + echo "cc=/usr/lib64/ccache/clang" >> "$GITHUB_OUTPUT" + else + echo "cxx=/usr/lib64/ccache/g++" >> "$GITHUB_OUTPUT" + echo "cc=/usr/lib64/ccache/gcc" >> "$GITHUB_OUTPUT" + fi + + # make sure that for every build the GitHub Actions cache key is unique + - name: Prepare CCache timestamp + id: ccache_cache_timestamp + run: + echo "timestamp=$(date +%Y-%m-%d-%H:%M:%S)-UTC" >> "$GITHUB_OUTPUT" + + - name: Cache CCache files + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + # cache is saved with this hierarchy + # dnf5-ccache-gcc-pr-main-111-2023-01-01-00:00:00-UTC + # -clang-pr-main-111-2023-01-01-00:00:00-UTC + key: ${{ github.event.repository.name }}-ccache-${{ matrix.compiler }}-pr-${{ github.base_ref }}-${{ github.event.number }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + # we want to match the latest cache prioritizing the same PR (e.g. + # 111), then the same branch (e.g. main), and, in all cases, we want + # to match the project-compiler (gcc,clang) match by date is not + # needed since caches are always matched by date. + # + # Example: dnf5-ccache-clang-pr-devel-111-2023-01-01-00:00:00-UTC will + # try to fetch cache from the same pr, 111 against devel. if no pr is + # found it will try to fetch any pr against devel branch. + restore-keys: | + ${{ github.event.repository.name }}-ccache-${{ matrix.compiler }}pr-${{ github.base_ref }}-${{ github.event.number }}- + ${{ github.event.repository.name }}-ccache-${{ matrix.compiler }}pr-${{ github.base_ref }}- + + - name: Zero CCache statistics + run: ccache -z + + - name: Configure DNF5 with CMake + run: CXX=${{ steps.configure_compiler.outputs.cxx }} CC=${{ steps.configure_compiler.outputs.cc }} cmake -S . -B build + + - name: Build DNF5 + run: cmake --build build + + - name: Display CCache statistics + run: ccache -s From 56fe5e8b3041ca9f8900346362be4ffd35940557 Mon Sep 17 00:00:00 2001 From: Nicola Sella Date: Thu, 14 Sep 2023 15:47:20 +0200 Subject: [PATCH 2/5] Add a dispatch workflow to clear all cache --- .github/workflows/clear-all-cache.yml | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/clear-all-cache.yml diff --git a/.github/workflows/clear-all-cache.yml b/.github/workflows/clear-all-cache.yml new file mode 100644 index 000000000..4972601df --- /dev/null +++ b/.github/workflows/clear-all-cache.yml @@ -0,0 +1,30 @@ +name: Clear cache + +on: + workflow_dispatch: + +permissions: + actions: write + +jobs: + clear-cache: + runs-on: ubuntu-latest + steps: + - name: Clear cache + uses: actions/github-script@v6 + with: + script: | + console.log("About to clear") + const caches = await github.rest.actions.getActionsCacheList({ + owner: context.repo.owner, + repo: context.repo.repo, + }) + for (const cache of caches.data.actions_caches) { + console.log(cache) + github.rest.actions.deleteActionsCacheById({ + owner: context.repo.owner, + repo: context.repo.repo, + cache_id: cache.id, + }) + } + console.log("Clear completed") From 18f631b558d428e60a8e8be7ab6dd14f5d47cc0d Mon Sep 17 00:00:00 2001 From: Nicola Sella Date: Thu, 14 Sep 2023 15:52:30 +0200 Subject: [PATCH 3/5] Add condition to run a clean build only when cache fails --- .github/workflows/ccache-build.yml | 3 ++- .github/workflows/ci.yml | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ccache-build.yml b/.github/workflows/ccache-build.yml index 4bc99090b..38728a63c 100644 --- a/.github/workflows/ccache-build.yml +++ b/.github/workflows/ccache-build.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: jobs: - build-dnf5: + cmake-build: name: CMake Build runs-on: ubuntu-latest container: @@ -69,6 +69,7 @@ jobs: run: CXX=${{ steps.configure_compiler.outputs.cxx }} CC=${{ steps.configure_compiler.outputs.cc }} cmake -S . -B build - name: Build DNF5 + id: ccache-build run: cmake --build build - name: Display CCache statistics diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7052df44..200baf1e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,9 @@ on: jobs: package-build: name: Package Build + needs: [ cmake-build ] + # If any of the jobs listed in needs return "failure" then run this job + if: ${{ always() && contains(needs.*.result, 'failure') }} runs-on: ubuntu-latest container: ghcr.io/rpm-software-management/dnf-ci-host strategy: From d97fd3cc5f0bbc3b25c4e48c4e554ace956b87fc Mon Sep 17 00:00:00 2001 From: Nicola Sella Date: Thu, 14 Sep 2023 17:31:42 +0200 Subject: [PATCH 4/5] Move Package Build steps to the same file --- .github/workflows/ci.yml | 49 ----------------- .../{ccache-build.yml => package-build.yml} | 55 ++++++++++++++++++- 2 files changed, 52 insertions(+), 52 deletions(-) rename .github/workflows/{ccache-build.yml => package-build.yml} (56%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 200baf1e5..9d2915ee4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,55 +7,6 @@ on: types: [checks_requested] jobs: - package-build: - name: Package Build - needs: [ cmake-build ] - # If any of the jobs listed in needs return "failure" then run this job - if: ${{ always() && contains(needs.*.result, 'failure') }} - runs-on: ubuntu-latest - container: ghcr.io/rpm-software-management/dnf-ci-host - strategy: - fail-fast: false # don't fail all matrix jobs if one of them fails - matrix: - compiler: ['', clang] # gcc is the default - steps: - - name: Check out ci-dnf-stack - uses: actions/checkout@v2 - with: - repository: rpm-software-management/ci-dnf-stack - - - name: Setup CI - id: setup-ci - uses: ./.github/actions/setup-ci - with: - copr-user: ${{secrets.COPR_USER}} - copr-api-token: ${{secrets.COPR_API_TOKEN}} - - - name: Check out sources - uses: actions/checkout@v2 - with: - path: gits/${{github.event.repository.name}} - ref: ${{github.event.pull_request.head.sha}} # check out the PR HEAD - fetch-depth: 0 - - - name: Rebase the pull request on target branch - run: | - pushd gits/${{github.event.repository.name}} - git config user.name github-actions - git config user.email github-actions@github.com - echo "Rebasing \"`git log --oneline -1`\" on ${{github.event.pull_request.base.ref}}: \"`git log --oneline -1 origin/${{github.event.pull_request.base.ref}}`\"" - git rebase origin/${{github.event.pull_request.base.ref}} - popd - - - name: Build in Copr - run: | - CHROOTS="fedora-37-x86_64, fedora-38-x86_64, fedora-rawhide-x86_64" - PROJECT_NAME="CI-libdnf5-pr${{github.event.pull_request.number}}" - if [[ -n "${{matrix.compiler}}" ]]; then - PROJECT_NAME+="-${{matrix.compiler}}" - fi - rpm-gitoverlay --gitdir=gits build-overlay -s overlays/dnf5-unstable rpm --with "${{matrix.compiler}}" copr --owner "${{steps.setup-ci.outputs.copr-user}}" --project "$PROJECT_NAME" --chroots "$CHROOTS" --delete-project-after-days=7 --additional-repos="copr://rpmsoftwaremanagement/dnf-nightly copr://rpmsoftwaremanagement/dnf5-unstable" - copr-build: name: Copr Build runs-on: ubuntu-latest diff --git a/.github/workflows/ccache-build.yml b/.github/workflows/package-build.yml similarity index 56% rename from .github/workflows/ccache-build.yml rename to .github/workflows/package-build.yml index 38728a63c..c0854d24f 100644 --- a/.github/workflows/ccache-build.yml +++ b/.github/workflows/package-build.yml @@ -1,12 +1,12 @@ --- -name: CCache Build +name: Package Build on: pull_request: workflow_dispatch: jobs: - cmake-build: - name: CMake Build + ccache-build: + name: CCache Build runs-on: ubuntu-latest container: image: fedora:rawhide @@ -74,3 +74,52 @@ jobs: - name: Display CCache statistics run: ccache -s + + rpm-gitoverlay-build: + name: Git Overlay Build + needs: [ ccache-build ] + # If any of the jobs listed in needs return "failure" then run this job + if: ${{ always() && contains(needs.*.result, 'failure') }} + runs-on: ubuntu-latest + container: ghcr.io/rpm-software-management/dnf-ci-host + strategy: + fail-fast: false # don't fail all matrix jobs if one of them fails + matrix: + compiler: ['', clang] # gcc is the default + steps: + - name: Check out ci-dnf-stack + uses: actions/checkout@v2 + with: + repository: rpm-software-management/ci-dnf-stack + + - name: Setup CI + id: setup-ci + uses: ./.github/actions/setup-ci + with: + copr-user: ${{secrets.COPR_USER}} + copr-api-token: ${{secrets.COPR_API_TOKEN}} + + - name: Check out sources + uses: actions/checkout@v2 + with: + path: gits/${{github.event.repository.name}} + ref: ${{github.event.pull_request.head.sha}} # check out the PR HEAD + fetch-depth: 0 + + - name: Rebase the pull request on target branch + run: | + pushd gits/${{github.event.repository.name}} + git config user.name github-actions + git config user.email github-actions@github.com + echo "Rebasing \"`git log --oneline -1`\" on ${{github.event.pull_request.base.ref}}: \"`git log --oneline -1 origin/${{github.event.pull_request.base.ref}}`\"" + git rebase origin/${{github.event.pull_request.base.ref}} + popd + + - name: Build in Copr + run: | + CHROOTS="fedora-37-x86_64, fedora-38-x86_64, fedora-rawhide-x86_64" + PROJECT_NAME="CI-libdnf5-pr${{github.event.pull_request.number}}" + if [[ -n "${{matrix.compiler}}" ]]; then + PROJECT_NAME+="-${{matrix.compiler}}" + fi + rpm-gitoverlay --gitdir=gits build-overlay -s overlays/dnf5-unstable rpm --with "${{matrix.compiler}}" copr --owner "${{steps.setup-ci.outputs.copr-user}}" --project "$PROJECT_NAME" --chroots "$CHROOTS" --delete-project-after-days=7 --additional-repos="copr://rpmsoftwaremanagement/dnf-nightly copr://rpmsoftwaremanagement/dnf5-unstable" From 7288c269824c88ba79fd54b10bce4afdd4498a85 Mon Sep 17 00:00:00 2001 From: Nicola Sella Date: Thu, 14 Sep 2023 17:40:22 +0200 Subject: [PATCH 5/5] Update workflow triggers --- .github/workflows/ci.yml | 3 +-- .github/workflows/package-build.yml | 3 ++- .github/workflows/pre-commit.yml | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d2915ee4..1e5c18c32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,7 @@ --- name: DNF 5 CI on: - pull_request_target: - types: [opened, reopened, synchronize] + pull_request: merge_group: types: [checks_requested] diff --git a/.github/workflows/package-build.yml b/.github/workflows/package-build.yml index c0854d24f..b440a127e 100644 --- a/.github/workflows/package-build.yml +++ b/.github/workflows/package-build.yml @@ -2,7 +2,8 @@ name: Package Build on: pull_request: - workflow_dispatch: + merge_group: + types: [checks_requested] jobs: ccache-build: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 0aa4a12aa..0a15ae7a7 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -2,6 +2,8 @@ name: Pre Commit on: pull_request: + merge_group: + types: [checks_requested] jobs: