Improve Bazel CI caching scheme.

.github/workflows/ci-bazel.yml

@@ -7,79 +7,169 @@ on:
     branches: [main]
   merge_group:
     branches: [main]
+  schedule:
+    # Run daily at 00:00 (https://crontab.guru/#0_0_*_*_*) to keep cache fresh.
+    - cron: "0 0 * * *"
+  # Allow manual triggering of the workflow.
+  # https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/manually-running-a-workflow
+  workflow_dispatch:
 
 # Cancel any preceding run on the pull request.
 concurrency:
   group: bazel-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
-  build_direct: # Build p4c directly.
+
+  format_bazel_files:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - run: tools/format-bazel-files.sh
+    - name: Check formatting diff
+      run: |
+        CHANGED_FILES="$(git diff-index --name-only HEAD --)"
+        if [[ -z "${CHANGED_FILES}" ]]; then
+          echo "Success: no formatting changes needed."
+          exit 0
+        fi
+        echo "Found formatting changes in the following files:"
+        echo "${CHANGED_FILES}"
+        echo ""
+        echo "Please run `tools/format-bazel-files.sh` to apply the changes."
+        exit 1
+
+  build-direct:  # Build p4c directly.
+    needs: format_bazel_files
     runs-on: ubuntu-latest
-    env:
-      BAZEL: bazelisk-linux-amd64
     steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - name: Mount bazel cache
-        uses: actions/cache@v4
-        with:
-          # See https://docs.bazel.build/versions/master/output_directories.html
-          path: "~/.cache/bazel"
-          # Create cash entry for every run (by using unique key derived from commit hash).
-          # See https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
-          key: ${{ runner.os }}-build-direct-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-build-direct-
-
-      - name: Install bazelisk
-        run: |
-          curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/$BAZEL"
-          chmod +x $BAZEL
-          sudo mv $BAZEL /usr/local/bin/bazel
-
-      - name: First, pass the lint test
-        run: bazel run //:buildifier_check
-
-      - name: Install Flex and Bison
-        run: sudo apt-get update && sudo apt-get install bison flex libfl-dev
-
-      - name: Build p4c
-        run: bazel build //... --verbose_failures
-
-  build_indirect: # Build 3rd party Bazel project depending on p4c as a subproject.
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Install Flex and Bison
+      run: sudo apt install bison flex libfl-dev
+
+    - name: Mount bazel cache
+      uses: actions/cache/restore@v4
+      with:
+        # See https://docs.bazel.build/versions/master/output_directories.html
+        path: "~/.cache/bazel"
+        key: bazel-direct-${{ hashFiles('**.bazel*', '**.bzl') }}
+        restore-keys: |
+          bazel-direct
+
+    - name: Save start time
+      uses: josStorer/get-current-time@v2
+      id: start-time
+      with:
+        # Unix timestamp: seconds since 1970.
+        format: X
+
+    - name: Build p4c directly
+      run: bazel build //... --verbose_failures
+
+    - name: Save end time
+      # Always needed for caching.
+      if: always()
+      uses: josStorer/get-current-time@v2
+      id: end-time
+      with:
+        # Unix timestamp: seconds since 1970.
+        format: X
+
+    - name: Calculate build duration
+      # Always needed for caching.
+      if: always()
+      run: |
+        START=${{ steps.start-time.outputs.formattedTime }}
+        END=${{ steps.end-time.outputs.formattedTime }}
+        DURATION=$(( $END - $START ))
+        echo "duration=$DURATION" | tee "$GITHUB_ENV"
+
+    - name: Compress cache
+      # Always needed for caching.
+      if: always()
+      run: rm -rf $(bazel info repository_cache)
+
+    - name: Save bazel cache
+      uses: actions/cache/save@v4
+      # Only create a new cache entry if we're on the main branch or the build
+      # takes > 10 mins.
+      #
+      # NOTE: Even though `always()` evaluates to true, and `true && x == x`,
+      # the `always() &&` prefix is not redundant! Calling `always()` has a
+      # side effect: it prevents this step from being skipped if a previous
+      # step failed.
+      if: always() && (github.ref_name == 'main' || env.duration > 600)
+      with:
+        path: "~/.cache/bazel"
+        key: bazel-direct-${{ hashFiles('**.bazel*', '**.bzl') }}-${{ github.run_id }}
+
+
+  build-indirect:  # Build 3rd party Bazel project depending on p4c as a subproject.
+    needs: format_bazel_files
     runs-on: ubuntu-latest
-    env:
-      BAZEL: bazelisk-linux-amd64
     steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - name: Mount bazel cache
-        uses: actions/cache@v4
-        with:
-          # See https://docs.bazel.build/versions/master/output_directories.html
-          path: "~/.cache/bazel"
-          # Create cash entry for every run (by using unique key derived from commit hash).
-          # See https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
-          key: ${{ runner.os }}-build-indirect-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-build-indirect-
-
-      - name: First, pass the lint test
-        run: bazel run //:buildifier_check
-
-      - name: Install bazelisk
-        run: |
-          curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/$BAZEL"
-          chmod +x $BAZEL
-          sudo mv $BAZEL /usr/local/bin/bazel
-
-      - name: Install Flex and Bison
-        run: sudo apt-get update && sudo apt-get install bison flex libfl-dev
-
-      - name: Build bazel/example
-        run: cd ./bazel/example && bazel build //... --verbose_failures
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Install Flex and Bison
+      run: sudo apt install bison flex libfl-dev
+
+    - name: Mount bazel cache
+      uses: actions/cache/restore@v4
+      with:
+        # See https://docs.bazel.build/versions/master/output_directories.html
+        path: "~/.cache/bazel"
+        key: bazel-indirect-${{ hashFiles('**.bazel*', '**.bzl') }}
+        restore-keys: |
+          bazel-indirect
+
+    - name: Save start time
+      uses: josStorer/get-current-time@v2
+      id: start-time
+      with:
+        # Unix timestamp: seconds since 1970.
+        format: X
+
+    - name: Build p4c via 3rd party project depending on p4c (`bazel/example`)
+      run: cd ./bazel/example && bazel build //... --verbose_failures
+
+    - name: Save end time
+      # Always needed for caching.
+      if: always()
+      uses: josStorer/get-current-time@v2
+      id: end-time
+      with:
+        # Unix timestamp: seconds since 1970.
+        format: X
+
+    - name: Calculate build duration
+      # Always needed for caching.
+      if: always()
+      run: |
+        START=${{ steps.start-time.outputs.formattedTime }}
+        END=${{ steps.end-time.outputs.formattedTime }}
+        DURATION=$(( $END - $START ))
+        echo "duration=$DURATION" | tee "$GITHUB_ENV"
+
+    - name: Compress cache
+      # Always needed for caching.
+      if: always()
+      run: rm -rf $(bazel info repository_cache)
+
+    - name: Save bazel cache
+      uses: actions/cache/save@v4
+      # Only create a new cache entry if we're on the main branch or the build
+      # takes > 10 mins.
+      #
+      # NOTE: Even though `always()` evaluates to true, and `true && x == x`,
+      # the `always() &&` prefix is not redundant! Calling `always()` has a
+      # side effect: it prevents this step from being skipped if a previous
+      # step failed.
+      if: always() && (github.ref_name == 'main' || env.duration > 600)
+      with:
+        path: "~/.cache/bazel"
+        key: bazel-indirect-${{ hashFiles('**.bazel*', '**.bzl') }}-${{ github.run_id }}

BUILD.bazel

@@ -1,7 +1,6 @@
 load("@rules_license//rules:license.bzl", "license")
-load("//:bazel/flex.bzl", "genlex")
 load("//:bazel/bison.bzl", "genyacc")
-load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier")
+load("//:bazel/flex.bzl", "genlex")
 
 package(
     default_applicable_licenses = ["//:license"],
@@ -212,14 +211,14 @@ cc_library(
         "midend/**/*.cpp",
         "control-plane/**/*.cpp",
     ]) + [
-        "frontends/parsers/p4/p4parser.cc",
-        "frontends/parsers/v1/v1parser.cc",
+        "backends/dpdk/dbprint-dpdk.cpp",
+        "backends/dpdk/printUtils.cpp",
+        "backends/dpdk/spec.cpp",
         "frontends/parsers/p4/p4lexer.cc",
+        "frontends/parsers/p4/p4parser.cc",
         "frontends/parsers/v1/v1lexer.cc",
+        "frontends/parsers/v1/v1parser.cc",
         "ir/ir-generated.cpp",
-        "backends/dpdk/spec.cpp",
-        "backends/dpdk/dbprint-dpdk.cpp",
-        "backends/dpdk/printUtils.cpp",
     ],
     textual_hdrs = glob([
         "ir/**/*.h",
@@ -484,21 +483,3 @@ cc_library(
         "@boost//:graph",
     ],
 )
-
-buildifier(
-    name = "buildifier_check",
-    diff_command = "diff -u",
-    exclude_patterns = [
-        "./build/*",
-    ],
-    lint_mode = "warn",
-    mode = "diff",
-)
-
-buildifier(
-    name = "buildifier_fix",
-    exclude_patterns = [
-        "./build/*",
-    ],
-    lint_mode = "fix",
-)

WORKSPACE.bazel

@@ -1,32 +1,5 @@
 workspace(name = "com_github_p4lang_p4c")
 
-# -- Load Buildifier  -----------------------------------------------------
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-# buildifier is written in Go and hence needs rules_go to be built.
-# See https://github.com/bazelbuild/rules_go for the up to date setup instructions.
-http_archive(
-    name = "io_bazel_rules_go",
-    sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996",
-    urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
-    ],
-)
-
-load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies")
-
-go_rules_dependencies()
-
-http_archive(
-    name = "com_github_bazelbuild_buildtools",
-    sha256 = "ae34c344514e08c23e90da0e2d6cb700fcd28e80c02e23e4d5715dddcb42f7b3",
-    strip_prefix = "buildtools-4.2.2",
-    urls = [
-        "https://github.com/bazelbuild/buildtools/archive/refs/tags/4.2.2.tar.gz",
-    ],
-)
-
 # -- Direct dependencies. ------------------------------------------------------
 
 load("//:bazel/p4c_deps.bzl", "p4c_deps")
@@ -70,3 +43,30 @@ grpc_extra_deps()
 load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
 
 rules_foreign_cc_dependencies()
+
+# -- Load Buildifier (for formatting) ------------------------------------------
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "buildifier_prebuilt",
+    integrity = "sha256-f4W2iKS1WOLZCZNAz7UQunF5+ClFT7qEI3C8z/tn1sw=",
+    # Newer versions of Buildifier enforce the use of Starlark versions of
+    # `proto_library`, `cc_{executable, library, proto_library, test}`, etc.
+    # We're not ready to make that migration yet, and use an older Buildifier
+    # version as a workaround.
+    strip_prefix = "buildifier-prebuilt-7.3.1",
+    urls = ["https://github.com/keith/buildifier-prebuilt/archive/7.3.1.tar.gz"],
+)
+
+load("@buildifier_prebuilt//:deps.bzl", "buildifier_prebuilt_deps")
+
+buildifier_prebuilt_deps()
+
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+
+bazel_skylib_workspace()
+
+load("@buildifier_prebuilt//:defs.bzl", "buildifier_prebuilt_register_toolchains")
+
+buildifier_prebuilt_register_toolchains()

bazel/example/WORKSPACE.bazel

@@ -12,33 +12,6 @@ local_repository(
     },
 )
 
-# -- Load Buildifier  -----------------------------------------------------
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-# buildifier is written in Go and hence needs rules_go to be built.
-# See https://github.com/bazelbuild/rules_go for the up to date setup instructions.
-http_archive(
-    name = "io_bazel_rules_go",
-    sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996",
-    urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
-    ],
-)
-
-load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies")
-
-go_rules_dependencies()
-
-http_archive(
-    name = "com_github_bazelbuild_buildtools",
-    sha256 = "ae34c344514e08c23e90da0e2d6cb700fcd28e80c02e23e4d5715dddcb42f7b3",
-    strip_prefix = "buildtools-4.2.2",
-    urls = [
-        "https://github.com/bazelbuild/buildtools/archive/refs/tags/4.2.2.tar.gz",
-    ],
-)
-
 load("@com_github_p4lang_p4c//:bazel/p4c_deps.bzl", "p4c_deps")
 
 p4c_deps()

tools/format-bazel-files.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Formats and fixes all Bazel files (*.bazel, *.bzl) using Buildifier.
+bazel run -- @buildifier_prebuilt//:buildifier \
+  --lint=fix \
+  -r $(bazel info workspace)