From c2ef6b5e1881bc12015188b0b59169884a991032 Mon Sep 17 00:00:00 2001 From: abikouo Date: Thu, 6 Jul 2023 13:14:31 +0200 Subject: [PATCH 1/3] - add safe-to-test to enforce security when CI runs on pull requests - add splitter job - we should not run the full test suite for each pull request --- .github/workflows/integration.yml | 100 +++++++++++++++++++---------- test-requirements.txt | 2 + tests/integration/requirements.yml | 11 ++++ 3 files changed, 78 insertions(+), 35 deletions(-) create mode 100644 tests/integration/requirements.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 82a1c2d5..68a2421b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -13,63 +13,92 @@ on: jobs: safe-to-test: + if: ${{ github.event.label.name == 'safe to test' }} || ${{ github.event.action != 'labeled' }} uses: ansible-network/github_actions/.github/workflows/safe-to-test.yml@main + splitter: + runs-on: ubuntu-latest + env: + source_dir: "cloud_awsops" + outputs: + test_targets: ${{ steps.splitter.outputs.test_targets }} + test_targets_json: ${{ steps.splitter.outputs.test_targets_json }} + test_jobs: ${{ steps.splitter.outputs.test_jobs }} + steps: + - name: Checkout collection + uses: actions/checkout@v3 + with: + path: ${{ env.source_dir }} + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: "0" + + - name: List changes for pull request + id: splitter + uses: ansible-network/github_actions/.github/actions/ansible_test_splitter@main + with: + collections_to_test: ${{ env.source_dir }} + total_jobs: 2 + + - name: Display splitter output + run: | + echo "test_targets=${{ steps.splitter.outputs.test_targets }}" + echo "test_targets_json=${{ steps.splitter.outputs.test_targets_json }}" + echo "test_jobs=${{ steps.splitter.outputs.test_jobs }}" + shell: bash test: - needs: - - safe-to-test runs-on: ubuntu-latest + needs: + - splitter + if: ${{ needs.splitter.outputs.test_targets != '' }} env: source: "./source" - aws_dir: "./amazon_aws" - crypto_dir: "./community.crypto" ansible_version: "stable-2.14" python_version: "3.9" + strategy: + fail-fast: false + matrix: + workflow-id: ${{ fromJson(needs.splitter.outputs.test_jobs) }} + name: "integration-${{ matrix.workflow-id }}" steps: + - name: Read target + id: read-targets + run: | + import json, os + with open(os.environ.get('GITHUB_OUTPUT'), "a", encoding="utf-8") as fh: + fh.write(f'ansible_test_targets={json.loads(os.environ.get("ALL_TEST_TARGETS")).get(os.environ.get("WORKFLOW_ID"))}\n') + shell: python + env: + ALL_TEST_TARGETS: ${{ needs.splitter.outputs.test_targets_json }} + WORKFLOW_ID: ${{ matrix.workflow-id }} + + - name: Display ansible test targets + run: | + echo "ansible_test_targets -> ${{ steps.read-targets.outputs.ansible_test_targets }}" + shell: bash + - name: Checkout collection uses: actions/checkout@v3 with: path: ${{ env.source }} ref: ${{ github.event.pull_request.head.sha }} + - name: Pre install collections dependencies first so the collection install does not + run: >- + ansible-galaxy collection install + --pre "-r${{ env.source }}/tests/integration/requirements.yml" + -p /home/runner/collections/ + shell: bash + - name: Build and install collection - id: install-collection + id: install uses: ansible-network/github_actions/.github/actions/build_install_collection@main with: install_python_dependencies: false source_path: ${{ env.source }} - # checkout and install 'amazon.aws' - - name: checkout ansible-collections/amazon.aws - uses: ansible-network/github_actions/.github/actions/checkout_dependency@main - with: - repository: ansible-collections/amazon.aws - path: ${{ env.amazon_aws }} - ref: main - - - name: install amazon.aws collection - uses: ansible-network/github_actions/.github/actions/build_install_collection@main - with: - install_python_dependencies: true - source_path: ${{ env.amazon_aws }} - - # checkout and install 'community.crypto' - - name: checkout ansible-collections/community.crypto - uses: ansible-network/github_actions/.github/actions/checkout_dependency@main - with: - repository: ansible-collections/community.crypto - path: ${{ env.crypto_dir }} - ref: main - - - name: install community.crypto collection - uses: ansible-network/github_actions/.github/actions/build_install_collection@main - with: - install_python_dependencies: false - source_path: ${{ env.crypto_dir }} - - name: Create AWS/sts session credentials uses: ansible-network/github_actions/.github/actions/ansible_aws_test_provider@main with: - collection_path: ${{ steps.install-collection.outputs.collection_path }} + collection_path: ${{ steps.install.outputs.collection_path }} ansible_core_ci_key: ${{ secrets.ANSIBLE_CORE_CI_KEY }} # we use raw git to create a repository in the tests @@ -83,7 +112,8 @@ jobs: - name: Run integration tests uses: ansible-network/github_actions/.github/actions/ansible_test_integration@main with: - collection_path: ${{ steps.install-collection.outputs.collection_path }} + collection_path: ${{ steps.install.outputs.collection_path }} python_version: ${{ env.python_version }} ansible_version: ${{ env.ansible_version }} ansible_test_requirement_files: 'test-requirements.txt' + ansible_test_targets: ${{ steps.read-targets.outputs.ansible_test_targets }} diff --git a/test-requirements.txt b/test-requirements.txt index 0d38bc5e..f5925c05 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1 +1,3 @@ cryptography +botocore>=1.25.0 +boto3>=1.22.0 diff --git a/tests/integration/requirements.yml b/tests/integration/requirements.yml new file mode 100644 index 00000000..92122fba --- /dev/null +++ b/tests/integration/requirements.yml @@ -0,0 +1,11 @@ +--- +collections: + - name: https://github.com/ansible-collections/amazon.aws.git + type: git + version: main + - name: https://github.com/ansible-collections/community.aws.git + type: git + version: main + - name: https://github.com/ansible-collections/community.crypto.git + type: git + version: main From b919d0774679cbd4bec27f2fff0cf4f36a31d8e7 Mon Sep 17 00:00:00 2001 From: abikouo Date: Tue, 11 Jul 2023 16:10:06 +0200 Subject: [PATCH 2/3] add community.general as required collection for tests --- tests/integration/requirements.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/requirements.yml b/tests/integration/requirements.yml index 92122fba..c037cca5 100644 --- a/tests/integration/requirements.yml +++ b/tests/integration/requirements.yml @@ -6,6 +6,5 @@ collections: - name: https://github.com/ansible-collections/community.aws.git type: git version: main - - name: https://github.com/ansible-collections/community.crypto.git - type: git - version: main + - community.crypto + - community.general From e799087802ffcfd8c5b2460075775f146400143b Mon Sep 17 00:00:00 2001 From: abikouo Date: Wed, 26 Jul 2023 19:06:08 +0200 Subject: [PATCH 3/3] add dependency with safe to test --- .github/workflows/integration.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 68a2421b..c4581a66 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -16,6 +16,8 @@ jobs: if: ${{ github.event.label.name == 'safe to test' }} || ${{ github.event.action != 'labeled' }} uses: ansible-network/github_actions/.github/workflows/safe-to-test.yml@main splitter: + needs: + - safe-to-test runs-on: ubuntu-latest env: source_dir: "cloud_awsops"