Run tier tests on Testing Farm

Add a reusable workflow that schedule tests on testing farm using the CLI. If the specified plan is not found, the test is ignored and passes. The tier tests are executed sequentially using the reusable workflow. The tests from the tier1.yaml plan are executed on a single distro, currently Fedora 37. If the 'tier' input is set as 'tier2', the tests from the tier2.yaml plan are executed in multiple distros, currently Fedora 37 and 38. The tests are triggered when a pull request is opened or reopened. Signed-off-by: Anderson Toshiyuki Sasaki <[email protected]>
RedHat-SP-Security · Mar 14, 2023 · 45d7f5e · 45d7f5e
1 parent 2a5d029
commit 45d7f5e
Show file tree

Hide file tree

Showing 5 changed files with 260 additions and 0 deletions.
diff --git a/.github/workflows/run-on-testing-farm.yaml b/.github/workflows/run-on-testing-farm.yaml
@@ -0,0 +1,55 @@
+name: Run plan on Testing Farm
+
+on:
+  workflow_call:
+    inputs:
+      distro:
+        required: true
+        type: string
+      plan:
+        required: true
+        type: string
+      pr_number:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      head_ref:
+        required: true
+        type: string
+    outputs:
+      result:
+        description: "Whether the tests passed"
+        value: ${{ jobs.run-test.outputs.result }}
+    secrets:
+      token:
+        required: true
+      testing_farm_token:
+        required: true
+
+jobs:
+  run-test:
+    runs-on: ubuntu-22.04
+    name: Run plan on Testing Farm
+    outputs:
+      result: ${{ steps.check_result.outputs.result }}
+    env:
+      DISTRO: ${{ inputs.distro }}
+      PLAN: ${{ inputs.plan }}
+      PR_NUMBER: ${{ inputs.pr_number }}
+      GH_TOKEN: ${{ secrets.token }}
+      TESTING_FARM_API_TOKEN: ${{ secrets.testing_farm_token }}
+      GIT_REF: ${{ inputs.head_ref }}
+      REPO: ${{ inputs.repo }}
+    steps:
+    - name: Print plan name
+      run: echo "Running plan ${{ inputs.plan }} on Testing Farm for PR ${{ inputs.pr_number }}"
+    - uses: actions/checkout@v3
+    - name: Install testing-farm script
+      run: pip3 -v install tft-cli
+    - name: Run tests on Testing Farm
+      run: testing-farm request --git-url "https://github.com/$REPO.git" --git-ref "$GIT_REF" --context pr_id=$PR_NUMBER --context distro=$DISTRO --arch x86_64 --compose $DISTRO --plan "$PLAN" 2>&1 | tee tt_output
+    - name: Check test result
+      id: check_result
+      run: if grep -q 'tests passed' tt_output; then echo "result=PASS" >> "GITHUB_OUTPUT"; elif grep -q "Did not find any plans" tt_output; then echo "result=IGNORE" >> "GITHUB_OUTPUT"; else echo "result=FAIL" >> "GITHUB_OUTPUT" && false; fi
diff --git a/.github/workflows/run-tests-on-pr.yaml b/.github/workflows/run-tests-on-pr.yaml
@@ -0,0 +1,30 @@
+name: Run tests for pull request
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  pr_test:
+    uses: ./.github/workflows/run-on-testing-farm.yaml
+    with:
+      distro: "fedora-37"
+      plan: "${{ github.event.number }}"
+      pr_number: ${{ github.event.number }}
+      repo: ${{ github.event.pull_request.head.repo.full_name }}
+      head_ref: ${{ github.event.pull_request.head.ref }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      testing_farm_token: ${{ secrets.TESTING_FARM_API_TOKEN }}
+  tiers:
+    needs: pr_test
+    uses: ./.github/workflows/tiers.yaml
+    with:
+      tier: "tier2"
+      pr_number: ${ github.event.issue.number}}
+      repo: ${{ github.event.repository.full_name }}
+      head_ref: ${{ needs.get_ref.outputs.head_ref }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      testing_farm_token: ${{ secrets.TESTING_FARM_API_TOKEN }}
+
diff --git a/.github/workflows/tiers.yaml b/.github/workflows/tiers.yaml
@@ -0,0 +1,61 @@
+name: Run tier tests
+
+on:
+  workflow_call:
+    inputs:
+      tier:
+        type: string
+      pr_number:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      head_ref:
+        required: true
+        type: string
+    outputs:
+      tier1-result:
+        description: "Whether Tier1 tests passed"
+        value: ${{ jobs.tier1.outputs.result }}
+      tier2-result:
+        description: "Whether Tier1 tests passed"
+        value: ${{ jobs.tier2.outputs.result }}
+    secrets:
+      token:
+        description: "Github access token"
+        required: true
+      testing_farm_token:
+        description: "Testing farm access token"
+        required: true
+
+jobs:
+  tier1:
+    name: Run Tier-1 tests
+    uses: ./.github/workflows/run-on-testing-farm.yaml
+    with:
+      distro: "fedora-37"
+      plan: "tier1"
+      pr_number: ${{ inputs.pr_number }}
+      repo: ${{ inputs.repo }}
+      head_ref: ${{ inputs.head_ref }}
+    secrets:
+      token: ${{ secrets.token }}
+      testing_farm_token: ${{ secrets.testing_farm_token }}
+  tier2:
+    name: Run Tier-2 tests
+    needs: tier1
+    if: contains(${{ inputs.tier }}, "tier2")
+    strategy:
+      matrix:
+        distro: ["fedora-37", "fedora-38"]
+    uses: ./.github/workflows/run-on-testing-farm.yaml
+    with:
+      distro: ${{ matrix.distro }}
+      plan: "tier2"
+      pr_number: ${{ inputs.pr_number }}
+      repo: ${{ inputs.repo }}
+      head_ref: ${{ inputs.head_ref }}
+    secrets:
+      token: ${{ secrets.token }}
+      testing_farm_token: ${{ secrets.testing_farm_token }}
diff --git a/plans/tier1.fmf b/plans/tier1.fmf
@@ -0,0 +1,51 @@
+summary:
+  Tests used by Packit/TFT CI on Github to test upstream keylime
+
+prepare:
+  - how: shell
+    script:
+     - systemctl disable --now dnf-makecache.service || true
+     - systemctl disable --now dnf-makecache.timer || true
+
+discover:
+  how: fmf
+  test: 
+   - /setup/configure_tpm_emulator
+   - /setup/install_upstream_keylime
+   - /setup/enable_keylime_debug_messages
+   # change IMA policy to simple and run one attestation scenario
+   # this is to utilize also a different parser
+   - /setup/configure_kernel_ima_module/ima_policy_simple
+   - /functional/basic-attestation-on-localhost
+
+execute:
+    how: tmt
+
+adjust:
+  - when: distro == centos-stream-9
+    prepare+:
+      - how: shell
+        script:
+         - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+
+  - when: distro == centos-stream-8
+    prepare+:
+      - how: shell
+        script:
+          - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+#          - yum config-manager --set-enabled powertools
+
+  - when: "distro == fedora-36 or distro == fedora-37"
+    prepare+:
+      - how: shell
+        order: 99
+        script:
+          - yum -y downgrade tpm2-tss
+
+  # discover step adjustments
+  # disable code coverage measurement everywhere except F37 and CS9
+  - when: distro != centos-stream-9 and distro != fedora-37
+    discover+:
+       test-:
+         - /setup/enable_keylime_coverage
+         - /setup/generate_coverage_report
diff --git a/plans/tier2.fmf b/plans/tier2.fmf
@@ -0,0 +1,63 @@
+summary:
+  Tests used by Packit/TFT CI on Github to test upstream keylime
+
+prepare:
+  - how: shell
+    script:
+     - systemctl disable --now dnf-makecache.service || true
+     - systemctl disable --now dnf-makecache.timer || true
+
+discover:
+  how: fmf
+  test: 
+   - /setup/configure_tpm_emulator
+   - /setup/install_upstream_keylime
+   - /setup/enable_keylime_debug_messages
+   - /setup/enable_keylime_coverage
+   # change IMA policy to simple and run one attestation scenario
+   # this is to utilize also a different parser
+   - /setup/configure_kernel_ima_module/ima_policy_simple
+   - /functional/basic-attestation-on-localhost
+   # now change IMA policy to signing and run all tests
+   - /setup/configure_kernel_ima_module/ima_policy_signing
+   - "/functional/.*"
+   - "/compatibility/.*"
+   - /update/basic-attestation-on-localhost/all
+   # now set zeromq as a default revocation notifier and test it
+   - /setup/configure_default_revocation_notifier/zeromq
+   - /functional/basic-attestation-on-localhost
+   # run upstream test suite
+   - /upstream/run_keylime_tests
+   - /setup/generate_coverage_report
+
+execute:
+    how: tmt
+
+adjust:
+  - when: distro == centos-stream-9
+    prepare+:
+      - how: shell
+        script:
+         - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+
+  - when: distro == centos-stream-8
+    prepare+:
+      - how: shell
+        script:
+          - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+#          - yum config-manager --set-enabled powertools
+
+  - when: "distro == fedora-36 or distro == fedora-37"
+    prepare+:
+      - how: shell
+        order: 99
+        script:
+          - yum -y downgrade tpm2-tss
+
+  # discover step adjustments
+  # disable code coverage measurement everywhere except F37 and CS9
+  - when: distro != centos-stream-9 and distro != fedora-37
+    discover+:
+       test-:
+         - /setup/enable_keylime_coverage
+         - /setup/generate_coverage_report