.github: Build only what is needed depending on changes

.github/changed-files.yaml

@@ -0,0 +1,84 @@
+docs:
+  - docs/**
+  - artwork/**
+  - .github/workflows/build-docs.yaml
+  - VERSION
+  - buildchain/buildchain/versions.py
+  - buildchain/buildchain/constants.py
+  - salt/metalk8s/defaults.yaml
+  - tools/lib-alert-tree/**
+  - tools/rule_extractor/alerting_rules.json
+  - tox.ini
+
+shell-ui:
+  - shell-ui/**
+  - .github/workflows/build-shell-ui.yaml
+  - VERSION
+
+unit-tests-ui: &ui
+  - ui/**
+
+unit-tests-shell-ui: &shell-ui
+  - shell-ui/**
+
+unit-tests-crd-client-generator:
+  - tools/crd-client-generator-js/**
+
+unit-tests-metalk8s-operator:
+  - operator/**
+
+unit-tests-storage-operator:
+  - storage-operator/**
+
+unit-tests-salt:
+  - salt/**
+  - tox.ini
+
+unit-tests-lib-alert-tree:
+  - tools/lib-alert-tree/**
+
+integration-tests-ui:
+  - *ui
+  - *shell-ui
+
+build:
+  - "**"
+  # Only consider pre-merge workflows
+  - "!.github/workflows/**"
+  - .github/workflows/pre-merge.yaml
+  - .github/workflows/build.yaml
+  - .github/workflows/e2e-tests.yaml
+  - .github/workflows/single-node-test.yaml
+  - .github/workflows/multi-node-test.yaml
+  # Filter out github stuff
+  - "!.github/CODEOWNERS"
+  - "!.github/changed-files.yaml"
+  - "!.github/dependabot.yaml"
+  - "!.github/ISSUE_TEMPLATE/**"
+  - "!.github/labeler.yaml"
+  - "!.github/PULL_REQUEST_TEMPLATE.md"
+  - "!.github/scripts/build-conformance-pr-content.sh"
+  - "!.github/scripts/stabilize_snapshot.py"
+  # Filter out docs
+  - "!docs/**"
+  - "!artwork/**"
+  - "!**/*.md"
+  # Filter out tools
+  - "!tools/**"
+  # Filter out charts
+  - "!charts/**"
+  # Filter out unit tests
+  - "!salt/tests"
+  - "!**/*_test.go"
+  # Filter out Solution operator lib
+  - "!solution-operator-lib/**"
+  # Filter out devX and tools
+  - "!.pre-commit-config.yaml"
+  - "!.devcontainer/**"
+  - "!.gitignore"
+  - "!LICENSE"
+  - "!README.md"
+  - "!CONTRIBUTING.md"
+  - "!CHANGELOG.md"
+  - "!Vagrantfile"
+  - "!vagrant_config.rb.example"

.github/workflows/pre-merge.yaml

@@ -25,13 +25,85 @@ on:
       - "q/*/**"
 
 jobs:
+  changed-files:
+    runs-on: ubuntu-24.04
+    outputs:
+      docs: ${{ steps.diff.outputs.docs_any_modified == 'true' }}
+      shell-ui: ${{ steps.diff.outputs.shell-ui_any_modified == 'true' }}
+      unit-tests-ui: ${{ steps.diff.outputs.unit-tests-ui_any_modified == 'true' }}
+      unit-tests-shell-ui: ${{ steps.diff.outputs.unit-tests-shell-ui_any_modified == 'true' }}
+      unit-tests-crd-client-generator: ${{ steps.diff.outputs.unit-tests-crd-client-generator_any_modified == 'true' }}
+      unit-tests-metalk8s-operator: ${{ steps.diff.outputs.unit-tests-metalk8s-operator_any_modified == 'true' }}
+      unit-tests-storage-operator: ${{ steps.diff.outputs.unit-tests-storage-operator_any_modified == 'true' }}
+      unit-tests-salt: ${{ steps.diff.outputs.unit-tests-salt_any_modified == 'true' }}
+      unit-tests-lib-alert-tree: ${{ steps.diff.outputs.unit-tests-lib-alert-tree_any_modified == 'true' }}
+      integration-tests-ui: ${{ steps.diff.outputs.integration-tests-ui_any_modified == 'true' }}
+      # If the workflow is triggered on a development branch let's always build
+      build: ${{ startsWith(github.ref_name, 'development/') || steps.diff.outputs.build_any_modified == 'true' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Get base sha
+        run: |
+          source VERSION
+          branch=development/${VERSION_MAJOR}.${VERSION_MINOR}
+          if [[ ! $(git ls-remote --heads origin $branch) ]]; then
+            branch=development/${VERSION_MAJOR}
+          fi
+          echo "BASE_SHA=$(git merge-base HEAD origin/$branch)" >> $GITHUB_ENV
+      - name: Get changed files
+        id: diff
+        uses: tj-actions/changed-files@v45
+        with:
+          base_sha: ${{ env.BASE_SHA }}
+          files_yaml_from_source_file: .github/changed-files.yaml
+      - name: List all changed files
+        env:
+          ALL_DOCS_CHANGED_FILES: ${{ steps.diff.outputs.docs_all_changed_files }}
+          ALL_SHELL_UI_CHANGED_FILES: ${{ steps.diff.outputs.shell-ui_all_changed_files }}
+          ALL_UNIT_TESTS_UI_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-ui_all_changed_files }}
+          ALL_UNIT_TESTS_SHELL_UI_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-shell-ui_all_changed_files }}
+          ALL_UNIT_TESTS_CRD_CLIENT_GENERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-crd-client-generator_all_changed_files }}
+          ALL_UNIT_TESTS_METALK8S_OPERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-metalk8s-operator_all_changed_files }}
+          ALL_UNIT_TESTS_STORAGE_OPERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-storage-operator_all_changed_files }}
+          ALL_UNIT_TESTS_SALT_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-salt_all_changed_files }}
+          ALL_UNIT_TESTS_LIB_ALERT_TREE_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-lib-alert-tree_all_changed_files }}
+          ALL_INTEGRATION_TESTS_UI_CHANGED_FILES: ${{ steps.diff.outputs.integration-tests-ui_all_changed_files }}
+          ALL_BUILD_CHANGED_FILES: ${{ steps.diff.outputs.build_all_changed_files }}
+        run: |
+          for type in docs shell-ui unit-tests-ui unit-tests-shell-ui unit-tests-crd-client-generator unit-tests-metalk8s-operator unit-tests-storage-operator unit-tests-salt unit-tests-lib-alert-tree integration-tests-ui build; do
+            echo "All $type changed files:"
+            changed_files_var=$(echo "ALL_${type^^}_CHANGED_FILES" | tr '-' '_')
+            echo "${!changed_files_var}"
+          done
+
   build:
+    needs: changed-files
+    if: needs.changed-files.outputs.build == 'true'
     uses: ./.github/workflows/build.yaml
     secrets: inherit
 
+  build-docs:
+    needs: changed-files
+    # Build of docs is triggered by "build" so do not re-trigger it
+    if: needs.changed-files.outputs.docs == 'true' && needs.changed-files.outputs.build != 'true'
+    uses: ./.github/workflows/build-docs.yaml
+    secrets: inherit
+
+  build-shell-ui:
+    needs: changed-files
+    # Build of shell-ui is triggered by "build" so do not re-trigger it
+    if: needs.changed-files.outputs.shell-ui == 'true' && needs.changed-files.outputs.build != 'true'
+    uses: ./.github/workflows/build-shell-ui.yaml
+    secrets: inherit
+
   e2e-tests:
     needs:
       - build
+      - changed-files
+    if: needs.changed-files.outputs.build == 'true'
     uses: ./.github/workflows/e2e-tests.yaml
     secrets: inherit
     with:
@@ -72,6 +144,8 @@ jobs:
         run: ./doit.sh lint
 
   unit_tests_ui:
+    needs: changed-files
+    if: needs.changed-files.outputs.unit-tests-ui == 'true'
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -113,6 +187,8 @@ jobs:
           source: artifacts
 
   unit_tests_shell_ui:
+    needs: changed-files
+    if: needs.changed-files.outputs.unit-tests-shell-ui == 'true'
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -144,6 +220,8 @@ jobs:
           npm run test --no-update-notifier
 
   unit_tests_crd_client_generator:
+    needs: changed-files
+    if: needs.changed-files.outputs.unit-tests-crd-client-generator == 'true'
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -173,6 +251,8 @@ jobs:
         run: npm run test --no-update-notifier
 
   unit_tests_metalk8s_operator:
+    needs: changed-files
+    if: needs.changed-files.outputs.unit-tests-metalk8s-operator == 'true'
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -201,6 +281,8 @@ jobs:
         run: make test
 
   unit_tests_storage_operator:
+    needs: changed-files
+    if: needs.changed-files.outputs.unit-tests-storage-operator == 'true'
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -229,6 +311,8 @@ jobs:
         run: make test
 
   unit_tests_salt:
+    needs: changed-files
+    if: needs.changed-files.outputs.unit-tests-salt == 'true'
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -244,6 +328,8 @@ jobs:
         run: tox -e unit-tests
 
   unit_tests_lib_alert_tree:
+    needs: changed-files
+    if: needs.changed-files.outputs.unit-tests-lib-alert-tree == 'true'
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -263,6 +349,8 @@ jobs:
         run: poetry run pytest -vv
 
   build_integration_container_nginx:
+    needs: changed-files
+    if: needs.changed-files.outputs.integration-tests-ui == 'true'
     runs-on: ubuntu-20.04
     permissions: write-all
     steps:
@@ -335,6 +423,8 @@ jobs:
     runs-on: ubuntu-20.04
     needs:
       - build_integration_container_nginx
+      - changed-files
+    if: needs.changed-files.outputs.integration-tests-ui == 'true'
     services:
       app:
         image: ghcr.io/${{ github.repository_owner }}/metalk8s-nginx-integration-tests:${{ github.sha }}
@@ -412,6 +502,9 @@ jobs:
   write-final-status:
     runs-on: ubuntu-20.04
     needs:
+      - changed-files
+      - build-docs
+      - build-shell-ui
       - e2e-tests
       - lint
       - unit_tests_ui