From d51c65953ad26f89cbb7491beb996784a409b558 Mon Sep 17 00:00:00 2001 From: Teddy Andrieux Date: Tue, 8 Oct 2024 16:19:41 +0000 Subject: [PATCH] .github: Build only what is needed depending on changes --- .github/changed-files.yaml | 84 +++++++++++++++++++++++++++++ .github/workflows/pre-merge.yaml | 93 ++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 .github/changed-files.yaml diff --git a/.github/changed-files.yaml b/.github/changed-files.yaml new file mode 100644 index 0000000000..140a11ecd6 --- /dev/null +++ b/.github/changed-files.yaml @@ -0,0 +1,84 @@ +docs: + - docs/** + - artwork/** + - .github/workflows/build-docs.yaml + - VERSION + - buildchain/buildchain/versions.py + - buildchain/buildchain/constants.py + - salt/metalk8s/defaults.yaml + - tools/lib-alert-tree/** + - tools/rule_extractor/alerting_rules.json + - tox.ini + +shell-ui: + - shell-ui/** + - .github/workflows/build-shell-ui.yaml + - VERSION + +unit-tests-ui: &ui + - ui/** + +unit-tests-shell-ui: &shell-ui + - shell-ui/** + +unit-tests-crd-client-generator: + - tools/crd-client-generator-js/** + +unit-tests-metalk8s-operator: + - operator/** + +unit-tests-storage-operator: + - storage-operator/** + +unit-tests-salt: + - salt/** + - tox.ini + +unit-tests-lib-alert-tree: + - tools/lib-alert-tree/** + +integration-tests-ui: + - *ui + - *shell-ui + +build: + - "**" + # Only consider pre-merge workflows + - "!.github/workflows/**" + - .github/workflows/pre-merge.yaml + - .github/workflows/build.yaml + - .github/workflows/e2e-tests.yaml + - .github/workflows/single-node-test.yaml + - .github/workflows/multi-node-test.yaml + # Filter out github stuff + - "!.github/CODEOWNERS" + - "!.github/changed-files.yaml" + - "!.github/dependabot.yaml" + - "!.github/ISSUE_TEMPLATE/**" + - "!.github/labeler.yaml" + - "!.github/PULL_REQUEST_TEMPLATE.md" + - "!.github/scripts/build-conformance-pr-content.sh" + - "!.github/scripts/stabilize_snapshot.py" + # Filter out docs + - "!docs/**" + - "!artwork/**" + - "!**/*.md" + # Filter out tools + - "!tools/**" + # Filter out charts + - "!charts/**" + # Filter out unit tests + - "!salt/tests" + - "!**/*_test.go" + # Filter out Solution operator lib + - "!solution-operator-lib/**" + # Filter out devX and tools + - "!.pre-commit-config.yaml" + - "!.devcontainer/**" + - "!.gitignore" + - "!LICENSE" + - "!README.md" + - "!CONTRIBUTING.md" + - "!CHANGELOG.md" + - "!Vagrantfile" + - "!vagrant_config.rb.example" diff --git a/.github/workflows/pre-merge.yaml b/.github/workflows/pre-merge.yaml index 53d26b6528..0be2854de4 100644 --- a/.github/workflows/pre-merge.yaml +++ b/.github/workflows/pre-merge.yaml @@ -25,13 +25,85 @@ on: - "q/*/**" jobs: + changed-files: + runs-on: ubuntu-24.04 + outputs: + docs: ${{ steps.diff.outputs.docs_any_modified == 'true' }} + shell-ui: ${{ steps.diff.outputs.shell-ui_any_modified == 'true' }} + unit-tests-ui: ${{ steps.diff.outputs.unit-tests-ui_any_modified == 'true' }} + unit-tests-shell-ui: ${{ steps.diff.outputs.unit-tests-shell-ui_any_modified == 'true' }} + unit-tests-crd-client-generator: ${{ steps.diff.outputs.unit-tests-crd-client-generator_any_modified == 'true' }} + unit-tests-metalk8s-operator: ${{ steps.diff.outputs.unit-tests-metalk8s-operator_any_modified == 'true' }} + unit-tests-storage-operator: ${{ steps.diff.outputs.unit-tests-storage-operator_any_modified == 'true' }} + unit-tests-salt: ${{ steps.diff.outputs.unit-tests-salt_any_modified == 'true' }} + unit-tests-lib-alert-tree: ${{ steps.diff.outputs.unit-tests-lib-alert-tree_any_modified == 'true' }} + integration-tests-ui: ${{ steps.diff.outputs.integration-tests-ui_any_modified == 'true' }} + # If the workflow is triggered on a development branch let's always build + build: ${{ startsWith(github.ref_name, 'development/') || steps.diff.outputs.build_any_modified == 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get base sha + run: | + source VERSION + branch=development/${VERSION_MAJOR}.${VERSION_MINOR} + if [[ ! $(git ls-remote --heads origin $branch) ]]; then + branch=development/${VERSION_MAJOR} + fi + echo "BASE_SHA=$(git merge-base HEAD origin/$branch)" >> $GITHUB_ENV + - name: Get changed files + id: diff + uses: tj-actions/changed-files@v45 + with: + base_sha: ${{ env.BASE_SHA }} + files_yaml_from_source_file: .github/changed-files.yaml + - name: List all changed files + env: + ALL_DOCS_CHANGED_FILES: ${{ steps.diff.outputs.docs_all_changed_files }} + ALL_SHELL_UI_CHANGED_FILES: ${{ steps.diff.outputs.shell-ui_all_changed_files }} + ALL_UNIT_TESTS_UI_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-ui_all_changed_files }} + ALL_UNIT_TESTS_SHELL_UI_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-shell-ui_all_changed_files }} + ALL_UNIT_TESTS_CRD_CLIENT_GENERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-crd-client-generator_all_changed_files }} + ALL_UNIT_TESTS_METALK8S_OPERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-metalk8s-operator_all_changed_files }} + ALL_UNIT_TESTS_STORAGE_OPERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-storage-operator_all_changed_files }} + ALL_UNIT_TESTS_SALT_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-salt_all_changed_files }} + ALL_UNIT_TESTS_LIB_ALERT_TREE_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-lib-alert-tree_all_changed_files }} + ALL_INTEGRATION_TESTS_UI_CHANGED_FILES: ${{ steps.diff.outputs.integration-tests-ui_all_changed_files }} + ALL_BUILD_CHANGED_FILES: ${{ steps.diff.outputs.build_all_changed_files }} + run: | + for type in docs shell-ui unit-tests-ui unit-tests-shell-ui unit-tests-crd-client-generator unit-tests-metalk8s-operator unit-tests-storage-operator unit-tests-salt unit-tests-lib-alert-tree integration-tests-ui build; do + echo "All $type changed files:" + changed_files_var=$(echo "ALL_${type^^}_CHANGED_FILES" | tr '-' '_') + echo "${!changed_files_var}" + done + build: + needs: changed-files + if: needs.changed-files.outputs.build == 'true' uses: ./.github/workflows/build.yaml secrets: inherit + build-docs: + needs: changed-files + # Build of docs is triggered by "build" so do not re-trigger it + if: needs.changed-files.outputs.docs == 'true' && needs.changed-files.outputs.build != 'true' + uses: ./.github/workflows/build-docs.yaml + secrets: inherit + + build-shell-ui: + needs: changed-files + # Build of shell-ui is triggered by "build" so do not re-trigger it + if: needs.changed-files.outputs.shell-ui == 'true' && needs.changed-files.outputs.build != 'true' + uses: ./.github/workflows/build-shell-ui.yaml + secrets: inherit + e2e-tests: needs: - build + - changed-files + if: needs.changed-files.outputs.build == 'true' uses: ./.github/workflows/e2e-tests.yaml secrets: inherit with: @@ -72,6 +144,8 @@ jobs: run: ./doit.sh lint unit_tests_ui: + needs: changed-files + if: needs.changed-files.outputs.unit-tests-ui == 'true' runs-on: ubuntu-20.04 steps: - name: Checkout @@ -113,6 +187,8 @@ jobs: source: artifacts unit_tests_shell_ui: + needs: changed-files + if: needs.changed-files.outputs.unit-tests-shell-ui == 'true' runs-on: ubuntu-20.04 steps: - name: Checkout @@ -144,6 +220,8 @@ jobs: npm run test --no-update-notifier unit_tests_crd_client_generator: + needs: changed-files + if: needs.changed-files.outputs.unit-tests-crd-client-generator == 'true' runs-on: ubuntu-20.04 defaults: run: @@ -173,6 +251,8 @@ jobs: run: npm run test --no-update-notifier unit_tests_metalk8s_operator: + needs: changed-files + if: needs.changed-files.outputs.unit-tests-metalk8s-operator == 'true' runs-on: ubuntu-20.04 defaults: run: @@ -201,6 +281,8 @@ jobs: run: make test unit_tests_storage_operator: + needs: changed-files + if: needs.changed-files.outputs.unit-tests-storage-operator == 'true' runs-on: ubuntu-20.04 defaults: run: @@ -229,6 +311,8 @@ jobs: run: make test unit_tests_salt: + needs: changed-files + if: needs.changed-files.outputs.unit-tests-salt == 'true' runs-on: ubuntu-20.04 steps: - name: Checkout @@ -244,6 +328,8 @@ jobs: run: tox -e unit-tests unit_tests_lib_alert_tree: + needs: changed-files + if: needs.changed-files.outputs.unit-tests-lib-alert-tree == 'true' runs-on: ubuntu-20.04 defaults: run: @@ -263,6 +349,8 @@ jobs: run: poetry run pytest -vv build_integration_container_nginx: + needs: changed-files + if: needs.changed-files.outputs.integration-tests-ui == 'true' runs-on: ubuntu-20.04 permissions: write-all steps: @@ -335,6 +423,8 @@ jobs: runs-on: ubuntu-20.04 needs: - build_integration_container_nginx + - changed-files + if: needs.changed-files.outputs.integration-tests-ui == 'true' services: app: image: ghcr.io/${{ github.repository_owner }}/metalk8s-nginx-integration-tests:${{ github.sha }} @@ -412,6 +502,9 @@ jobs: write-final-status: runs-on: ubuntu-20.04 needs: + - changed-files + - build-docs + - build-shell-ui - e2e-tests - lint - unit_tests_ui