From 098e139ad1480bea92ba42f2873429aa3d5f734f Mon Sep 17 00:00:00 2001 From: Anderson Toshiyuki Sasaki Date: Tue, 14 Mar 2023 14:54:26 +0100 Subject: [PATCH] Run tier tests on Testing Farm Add a reusable workflow that schedule tests on testing farm using the CLI. If the specified plan is not found, the test is ignored and passes. The tier tests are executed sequentially using the reusable workflow. The tests from the tier1.yaml plan are executed on a single distro, currently Fedora 37. If the 'tier' input is set as 'tier2', the tests from the tier2.yaml plan are executed in multiple distros, currently Fedora 37 and 38. The tests are triggered when a pull request is opened or reopened. Signed-off-by: Anderson Toshiyuki Sasaki --- .../actions/run-on-testing-farm/action.yaml | 265 ++++++++++++++++++ .../run-on-testing-farm/src/summary.py | 59 ++++ .github/workflows/pr-test.yaml | 57 ++++ .github/workflows/register_pr.yaml | 24 ++ .github/workflows/tiers.yaml | 79 ++++++ .github/workflows/trigger-on-pr.yaml | 85 ++++++ .packit.yaml | 12 - plans/tier1.fmf | 49 ++++ plans/tier2.fmf | 58 ++++ 9 files changed, 676 insertions(+), 12 deletions(-) create mode 100644 .github/actions/run-on-testing-farm/action.yaml create mode 100644 .github/actions/run-on-testing-farm/src/summary.py create mode 100644 .github/workflows/pr-test.yaml create mode 100644 .github/workflows/register_pr.yaml create mode 100644 .github/workflows/tiers.yaml create mode 100644 .github/workflows/trigger-on-pr.yaml delete mode 100644 .packit.yaml create mode 100644 plans/tier1.fmf create mode 100644 plans/tier2.fmf diff --git a/.github/actions/run-on-testing-farm/action.yaml b/.github/actions/run-on-testing-farm/action.yaml new file mode 100644 index 00000000..9945c6f6 --- /dev/null +++ b/.github/actions/run-on-testing-farm/action.yaml @@ -0,0 +1,265 @@ +name: 'Run plan on Testing Farm' +description: 'Run a test plan on Testing Farm' + +inputs: + api_url: + description: 'A Testing Farm server URL' + required: false + default: 'https://api.dev.testing-farm.io/v0.1' + distro: + description: 'The distro identifier' + required: true + type: string + compose: + description: 'The compose to use' + required: true + type: string + arch: + description: 'Define an architecture for testing environment. Default: x86_64' + required: false + default: 'x86_64' + plan: + description: 'The name of the plan to run' + required: true + type: string + pr_number: + description: 'The number of the PR for which the tests are executed' + required: true + type: string + pr_url: + description: 'The URL to the PR for which the tests are executed' + required: true + type: string + repo: + description: 'The repository complete name (i.e. owner/repo)' + required: true + type: string + branch: + description: 'The branch to test' + required: true + type: string + head_sha: + description: 'The HEAD commit to test' + required: true + type: string + token: + description: 'The Github access token' + required: true + testing_farm_token: + description: 'The testing farm access token' + required: true + +outputs: + request_id: + description: 'An ID of a scheduled testing farm request' + value: ${{ steps.sched_test.outputs.req_id }} + request_url: + description: 'An url of a scheduled testing farm request' + value: ${{ steps.artifacts_url.outputs.url }} +secrets: + +runs: + using: "composite" + steps: + - name: Install mandatory packages + run: | + sudo apt update && sudo apt -y install curl jq libxml2-utils + pip3 -v install tft-cli + shell: bash + + - name: Set artifacts url + id: artifacts_url + run: | + url="https://artifacts.dev.testing-farm.io" + echo "url=$url" >> $GITHUB_OUTPUT + shell: bash + + - name: Create status pull request status name + id: create_status_name + run: echo "name=Plan ${{ inputs.plan }} on ${{ inputs.distro }} ${{ inputs.arch }} for [PR ${{ inputs.pr_number }}](${{ inputs.pr_url }})" >> $GITHUB_OUTPUT + shell: bash + + - name: Schedule a test on Testing Farm + id: sched_test + run: | + cat << EOF > request.json + { + "api_key": "${{ inputs.testing_farm_token }}", + "test": { + "fmf": { + "url": "https://github.com/${{ inputs.repo }}.git", + "ref": "${{ inputs.branch }}", + "name": "${{ inputs.plan }}" + } + }, + "environments": [{ + "arch": "${{ inputs.arch }}", + "os": { + "compose": "${{ inputs.compose }}" + }, + "tmt": { + "context": { + "pr_id": "${{ inputs.pr_number }}", + "distro": "${{ inputs.distro }}" + } + } + }] + } + EOF + + # DEBUG + jq < request.json + + curl ${{ inputs.api_url }}/requests \ + --data @request.json \ + --header "Content-Type: application/json" \ + --output response.json + + # DEBUG + jq < response.json + + req_id=$(jq -r .id response.json) + echo "req_id=$req_id" >> $GITHUB_OUTPUT + shell: bash + + - name: Switch pull request state to running + id: running + run: | + # Create running.json file for query, whether job is finished or not. + cat << EOF > running.json + { + "sha": "${{ inputs.head_sha }}", + "state": "pending", + "context": "Testing Farm - ${{ steps.create_status_name.outputs.name }}", + "description": "Build started", + "target_url": "${{ steps.artifacts_url.outputs.url }}/${{ steps.sched_test.outputs.req_id }}" + } + EOF + + # Update GitHub status description to 'Build started' + curl -X POST \ + -H "Authorization: Bearer ${{ inputs.token }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/${{ inputs.branch }} \ + --data @running.json > update.json + shell: bash + + - name: Check if scheduled test is still running + id: still_running + run: | + CMD=${{ inputs.api_url }}/requests/${{ steps.sched_test.outputs.req_id }} + curl $CMD > job.json + + # DEBUG + jq < job.json + + state=$(jq -r .state job.json) + while [ "$state" == "running" ] || [ "$state" == "new" ] || [ "$state" == "pending" ] || [ "$state" == "queued" ]; do + sleep 30 + curl $CMD > job.json + state=$(jq -r .state job.json) + done + shell: bash + + - name: Get final state of Testing Farm scheduled request + id: final_state + run: | + curl ${{ inputs.api_url }}/requests/${{ steps.sched_test.outputs.req_id }} > job.json + + # DEBUG + jq < job.json + + content=`cat job.json` + echo "result=$content" >> $GITHUB_OUTPUT + echo "not_found=Did not find any plans" >> $GITHUB_OUTPUT + shell: bash + + - name: Process result and ignore if plan is not found + id: check_result + run: | + state=${{ fromJson(steps.final_state.outputs.result).state }} + overall=${{ fromJson(steps.final_state.outputs.result).result.overall }} + can_ignore=${{ contains(fromJson(steps.final_state.outputs.result).result.summary, steps.final_state.outputs.not_found) }} + echo "can_ignore=$can_ignore" + new_state="success" + infra_error="" + log="" + echo "State is $state and result is: $overall" + if [[ "$state" == "complete" ]]; then + if [[ "$overall" != "passed" ]]; then + if [[ "$can_ignore" == "true" ]]; then + echo "The test plan was not found, ignoring error." + new_state="ignored" + else + new_state="failed" + fi + fi + else + # Mark job in case of infrastructure issues. Report to Testing Farm team + infra_error="- Infra problems" + new_state="failed" + log="pipeline.log" + fi + echo "New State is: $new_state" + echo "Infra state is: $infra_error" + echo "FINAL_STATE=$new_state" >> $GITHUB_OUTPUT + echo "INFRA_STATE=$infra_error" >> $GITHUB_OUTPUT + echo "RESULT_SUMMARY=$result_summary" >> $GITHUB_OUTPUT + echo "LOG=$log" >> $GITHUB_OUTPUT + shell: bash + + - name: Switch pull request GitHub status to final state + if: ${{ inputs.update_pull_request_status == 'true' }} + run: | + cat << EOF > final_request.json + { + "sha": "${{ inputs.head_sha }}", + "state": "${{ steps.check_result.outputs.FINAL_STATE }}", + "context": "Testing Farm - ${{ steps.create_status_name.outputs.name }}", + "description": "Build finished ${{ steps.check_result.outputs.INFRA_STATE }}", + "target_url": "${{ steps.artifacts_url.outputs.url }}/${{ steps.sched_test.outputs.req_id }}/${{ steps.check_result.outputs.LOG }}" + } + EOF + # Switch Github status to proper state + curl -X POST -H "Authorization: Bearer ${{ inputs.token }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/${{ inputs.branch }} \ + --data @final_request.json > final_response.json + shell: bash + + - name: Create github summary + shell: bash + run: | + if [ -z "${{ steps.check_result.outputs.INFRA_STATE }}" ]; then + infra_state=OK + else + infra_state=Failed + fi + tfaga_summary_header="Summary for ${{ steps.create_status_name.outputs.name }}:", + if ! cat "$GITHUB_STEP_SUMMARY" | grep "$tfaga_summary_header" > /dev/null; then + echo "$tfaga_summary_header" >> "$GITHUB_STEP_SUMMARY" + echo "| Compose | Arch | Infrastructure State | Test result | Link to logs |" >> "$GITHUB_STEP_SUMMARY" + echo "|---------|------|----------------------|-------------|--------------|" >> "$GITHUB_STEP_SUMMARY" + fi + echo "|${{ inputs.distro }}|${{ inputs.arch }}| $infra_state |"\ + " ${{ steps.check_result.outputs.FINAL_STATE }}|"\ + "${{ steps.artifacts_url.outputs.url }}/${{ steps.sched_test.outputs.req_id }}|" >> "$GITHUB_STEP_SUMMARY" + + - name: Show tests details + shell: bash + run: | + xunit='${{ fromJson(steps.final_state.outputs.result).result.xunit }}' + if [[ ! -z "$xunit" ]]; then + echo "$xunit" > xunit.xml + xmllint --format xunit.xml > results.xml + python3 .github/actions/run-on-testing-farm/src/summary.py results.xml >> $GITHUB_STEP_SUMMARY + fi + + - name: Exit with error in case of failure in test + shell: bash + run: | + final_state="${{ steps.check_result.outputs.FINAL_STATE }}" + if [ "$final_state" == "failed" ]; then + exit 1 + fi + diff --git a/.github/actions/run-on-testing-farm/src/summary.py b/.github/actions/run-on-testing-farm/src/summary.py new file mode 100644 index 00000000..5ff2da2a --- /dev/null +++ b/.github/actions/run-on-testing-farm/src/summary.py @@ -0,0 +1,59 @@ +import argparse +import xml.etree.ElementTree as ET + + +def parse_result(results): + + tree = ET.parse(results) + root = tree.getroot() + + for testsuite in root.findall('testsuite'): + name = testsuite.get('name') + overall = testsuite.get('result') + num_tests = testsuite.get('tests') + formatted_logs = "No logs found" + logs = testsuite.find('logs') + + if logs: + logs_list = map(lambda x: f"[{x[0]}]({x[1]})", + map(lambda l: (l.get('name'), l.get('href')), logs.findall('log'))) + formatted_logs = ", ".join(logs_list) + # Print 1 line table with overall result as the summary + print("|Test suite| Result | Logs |") + print("|----------|--------|------|") + print(f"| {name} | {overall} | {formatted_logs}|") + print(f"
Test details") + print("

\n") + print("| Test | Result | Logs |") + print("|------|--------|------|") + for testcase in testsuite.findall('testcase'): + name = testcase.get('name') + result = testcase.get('result') + formatted_logs = "No logs found" + logs = testcase.find('logs') + if logs: + logs_list = map(lambda x: f"[{x[0]}]({x[1]})", + map(lambda l: (l.get('name'), l.get('href')), logs.findall('log'))) + formatted_logs = ", ".join(logs_list) + print(f"| {name} | {result} | {formatted_logs}|") + print("\n

") + print("
") + + +def main(): + parser = argparse.ArgumentParser( + description="Print results in markdown format") + + parser.add_argument( + 'filename', help="The file containing the xunit xml file returned by Testing Farm") + + args = parser.parse_args() + + if args.filename: + parse_result(args.filename) + else: + print("No results to show") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/pr-test.yaml b/.github/workflows/pr-test.yaml new file mode 100644 index 00000000..86347fea --- /dev/null +++ b/.github/workflows/pr-test.yaml @@ -0,0 +1,57 @@ +name: PR test + +on: + workflow_call: + inputs: + distro: + required: true + type: string + arch: + required: true + type: string + compose: + required: true + type: string + plan: + required: true + type: string + pr_number: + required: true + type: string + pr_url: + required: true + type: string + repo: + required: true + type: string + branch: + required: true + type: string + head_sha: + required: true + type: string + secrets: + token: + required: true + testing_farm_token: + required: true + +jobs: + run-test: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - name: "Plan ${{ inputs.plan }} on ${{ inputs.distro }} ${{ inputs.arch }} for [PR ${{ inputs.pr_number }}](${{ inputs.pr_url }})" + uses: ./.github/actions/run-on-testing-farm + with: + distro: ${{ inputs.distro }} + compose: ${{ inputs.compose }} + arch: ${{ inputs.arch }} + plan: ${{ inputs.plan }} + pr_number: ${{ inputs.pr_number }} + pr_url: ${{ inputs.pr_url }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + head_sha: ${{ inputs.head_sha }} + token: ${{ secrets.token }} + testing_farm_token: ${{ secrets.testing_farm_token }} diff --git a/.github/workflows/register_pr.yaml b/.github/workflows/register_pr.yaml new file mode 100644 index 00000000..83e97b5a --- /dev/null +++ b/.github/workflows/register_pr.yaml @@ -0,0 +1,24 @@ +name: Register PR + +on: + pull_request: + types: [opened, reopened, synchronize] + +jobs: + register: + runs-on: ubuntu-22.04 + - name: Save PR info + run: | + mkdir -p ./pr + echo "{ + \"pr_number\": \"${{ github.event.number }}\", + \"pr_url\": \"${{ github.event.pull_request.issue_url }}\", + \"repo\": \"${{ github.event.pull_request.head.repo.full_name }}\", + \"branch\": \"${{ github.event.pull_request.head.ref }}\", + \"head_sha\": \"${{ github.event.pull_request.head.sha }}\" + }" > ./pr/pr_data.json + - uses: actions/upload-artifact@v3 + with: + name: pr_data + path: pr/ + diff --git a/.github/workflows/tiers.yaml b/.github/workflows/tiers.yaml new file mode 100644 index 00000000..edbdc0b9 --- /dev/null +++ b/.github/workflows/tiers.yaml @@ -0,0 +1,79 @@ +name: Tier tests + +on: + workflow_call: + inputs: + tier: + type: string + pr_number: + required: true + type: string + pr_url: + required: true + type: string + repo: + required: true + type: string + branch: + required: true + type: string + head_sha: + required: true + type: string + secrets: + token: + required: true + testing_farm_token: + required: true + +jobs: + tier1: + runs-on: ubuntu-22.04 + name: 'Tier 1' + steps: + - uses: actions/checkout@v3 + - name: "Tier" + uses: ./.github/actions/run-on-testing-farm + with: + distro: "fedora-37" + compose: "Fedora-37" + arch: "x86_64" + plan: "tier1" + pr_number: ${{ inputs.pr_number }} + pr_url: ${{ inputs.pr_url }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + head_sha: ${{ inputs.head_sha }} + token: ${{ secrets.token }} + testing_farm_token: ${{ secrets.testing_farm_token }} + tier2: + runs-on: ubuntu-22.04 + name: 'Tier 2' + needs: tier1 + if: contains(${{ inputs.tier }}, "tier2") + strategy: + fail-fast: false + matrix: + compose: ["Fedora-37", "Fedora-38"] + arch: ["x86_64"] + steps: + - uses: actions/checkout@v3 + - name: "Resolve distro" + id: resolve_distro + shell: bash + run: | + echo "distro=${{ matrix.compose }}" | tr '[:upper:]' '[:lower:]' >> $GITHUB_OUTPUT + - name: "Plan tier2 on ${{ steps.resolve_distro.outputs.distro }} ${{ matrix.arch }} for [PR ${{ inputs.pr_number }}](${{ inputs.pr_url }})" + uses: ./.github/actions/run-on-testing-farm + with: + distro: ${{ steps.resolve_distro.outputs.distro }} + compose: ${{ matrix.distro }} + arch: ${{ matrix.arch }} + plan: "tier2" + pr_number: ${{ inputs.pr_number }} + pr_url: ${{ inputs.pr_url }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + head_sha: ${{ inputs.head_sha }} + token: ${{ secrets.token }} + testing_farm_token: ${{ secrets.testing_farm_token }} diff --git a/.github/workflows/trigger-on-pr.yaml b/.github/workflows/trigger-on-pr.yaml new file mode 100644 index 00000000..690f6236 --- /dev/null +++ b/.github/workflows/trigger-on-pr.yaml @@ -0,0 +1,85 @@ +name: Testing Farm + +on: + workflow_run: + workflows: [Register PR] + types: [completed] + +jobs: + get_data: + outputs: + pr_number: ${{ steps.recover.outputs.pr_number }} + pr_url: ${{ steps.recover.outputs.pr_url}} + repo: ${{ steps.recover.outputs.repo }} + branch: ${{ steps.recover.outputs.branch }} + head_sha: ${{ steps.recover.outputs.head_sha }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - name: 'Download artifact' + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "pr_data" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); + - name: 'Unzip' + id: unzip + shell: bash + run: | + unzip pr_data.zip + echo "data=\"$(cat pr/pr_data.json)\"" >> $GITHUB_OUTPUT + - name: 'Recover data' + id: recover + shell: bash + run: | + echo "pr_number=${{ fromJson(steps.unzip.outputs.data).pr_number }}" >> $GITHUB_OUTPUT + echo "pr_url=${{ fromJson(steps.unzip.outputs.data).pr_url }}" >> $GITHUB_OUTPUT + echo "repo=${{ fromJson(steps.unzip.outputs.data).repo }}" >> $GITHUB_OUTPUT + echo "branch=${{ fromJson(steps.unzip.outputs.data).branch }}" >> $GITHUB_OUTPUT + echo "head_sha=${{ fromJson(steps.unzip.outputs.data).head_sha }}" >> $GITHUB_OUTPUT + pr_test: + needs: get_data + name: PR specific test + uses: ./.github/workflows/pr-test.yaml + with: + distro: "fedora-37" + compose: "Fedora-37" + arch: "x86_64" + plan: ${{ needs.get_data.outputs.pr_number }} + pr_number: ${{ needs.get_data.outputs.pr_number }} + pr_url: ${{ needs.get_data.outputs.pr_url }} + repo: ${{ needs.get_data.outputs.repo }} + branch: ${{ needs.get_data.outputs.branch }} + head_sha: ${{ needs.get_data.outputs.head_sha }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + testing_farm_token: ${{ secrets.TESTING_FARM_API_TOKEN }} + tier: + needs: [get_data, pr_test] + name: Tier tests + uses: ./.github/workflows/tiers.yaml + with: + tier: "tier2" + pr_number: ${{ needs.get_data.outputs.pr_number }} + pr_url: ${{ needs.get_data.outputs.pr_url }} + repo: ${{ needs.get_data.outputs.repo }} + branch: ${{ needs.get_data.outputs.branch }} + head_sha: ${{ needs.get_data.outputs.head_sha }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + testing_farm_token: ${{ secrets.TESTING_FARM_API_TOKEN }} + diff --git a/.packit.yaml b/.packit.yaml deleted file mode 100644 index 2fb5a38b..00000000 --- a/.packit.yaml +++ /dev/null @@ -1,12 +0,0 @@ -jobs: -- job: tests - trigger: pull_request - targets: - - fedora-stable - - centos-stream-9-x86_64 - skip_build: true - tf_extra_params: - environments: - - tmt: - context: - target_PR_branch: "main" diff --git a/plans/tier1.fmf b/plans/tier1.fmf new file mode 100644 index 00000000..bd20327f --- /dev/null +++ b/plans/tier1.fmf @@ -0,0 +1,49 @@ +summary: + Tests for keylime rust agent + +environment+: + TPM_BINARY_MEASUREMENTS: /var/tmp/binary_bios_measurements + KEYLIME_RUST_CODE_COVERAGE: 0 + RPM_AGENT_COVERAGE: 0 + +context+: + agent: rust + +prepare: + - how: shell + script: + - systemctl disable --now dnf-makecache.service || true + - systemctl disable --now dnf-makecache.timer || true + +adjust: + - when: distro == centos-stream-9 + prepare+: + - how: shell + script: + - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + - yum config-manager --set-enabled crb + - when: distro == centos-stream-8 + enabled: false + # disable temporarily on Rawhide since latest Rust agent cannot be compiled + - when: distro == fedora-rawhide + enabled: false + + - when: "distro == fedora-36 or distro == fedora-37" + prepare+: + - how: shell + order: 99 + script: + - yum -y downgrade tpm2-tss + +discover: + how: fmf + test: + - /setup/configure_tpm_emulator + - /setup/install_upstream_keylime + - /setup/install_upstream_rust_keylime + - /setup/enable_keylime_debug_messages + - /setup/configure_kernel_ima_module/ima_policy_signing + - /functional/basic-attestation-on-localhost + +execute: + how: tmt diff --git a/plans/tier2.fmf b/plans/tier2.fmf new file mode 100644 index 00000000..e8a5e1dc --- /dev/null +++ b/plans/tier2.fmf @@ -0,0 +1,58 @@ +summary: + Tests for keylime rust agent + +environment+: + TPM_BINARY_MEASUREMENTS: /var/tmp/binary_bios_measurements + KEYLIME_RUST_CODE_COVERAGE: 1 + RPM_AGENT_COVERAGE: 0 + +context+: + agent: rust + +prepare: + - how: shell + script: + - systemctl disable --now dnf-makecache.service || true + - systemctl disable --now dnf-makecache.timer || true + +adjust: + - when: distro == centos-stream-9 + prepare+: + - how: shell + script: + - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + - yum config-manager --set-enabled crb + - when: distro == centos-stream-8 + enabled: false + # disable temporarily on Rawhide since latest Rust agent cannot be compiled + - when: distro == fedora-rawhide + enabled: false + + - when: "distro == fedora-36 or distro == fedora-37" + prepare+: + - how: shell + order: 99 + script: + - yum -y downgrade tpm2-tss + # disable code coverage measurement everywhere except F37 and CS9 + - when: distro != centos-stream-9 and distro != fedora-37 + environment+: + KEYLIME_RUST_CODE_COVERAGE: 0 + discover+: + test-: + - /setup/generate_usptream_rust_keylime_code_coverage + +discover: + how: fmf + test: + - /setup/configure_tpm_emulator + - /setup/install_upstream_keylime + - /setup/install_upstream_rust_keylime + - /setup/enable_keylime_debug_messages + - /setup/configure_kernel_ima_module/ima_policy_signing + - "/functional/.*" + - /upstream/run_rust_keylime_tests + - /setup/generate_usptream_rust_keylime_code_coverage + +execute: + how: tmt