diff --git a/.github/actions/run-on-testing-farm/action.yaml b/.github/actions/run-on-testing-farm/action.yaml new file mode 100644 index 00000000..9945c6f6 --- /dev/null +++ b/.github/actions/run-on-testing-farm/action.yaml @@ -0,0 +1,265 @@ +name: 'Run plan on Testing Farm' +description: 'Run a test plan on Testing Farm' + +inputs: + api_url: + description: 'A Testing Farm server URL' + required: false + default: 'https://api.dev.testing-farm.io/v0.1' + distro: + description: 'The distro identifier' + required: true + type: string + compose: + description: 'The compose to use' + required: true + type: string + arch: + description: 'Define an architecture for testing environment. Default: x86_64' + required: false + default: 'x86_64' + plan: + description: 'The name of the plan to run' + required: true + type: string + pr_number: + description: 'The number of the PR for which the tests are executed' + required: true + type: string + pr_url: + description: 'The URL to the PR for which the tests are executed' + required: true + type: string + repo: + description: 'The repository complete name (i.e. owner/repo)' + required: true + type: string + branch: + description: 'The branch to test' + required: true + type: string + head_sha: + description: 'The HEAD commit to test' + required: true + type: string + token: + description: 'The Github access token' + required: true + testing_farm_token: + description: 'The testing farm access token' + required: true + +outputs: + request_id: + description: 'An ID of a scheduled testing farm request' + value: ${{ steps.sched_test.outputs.req_id }} + request_url: + description: 'An url of a scheduled testing farm request' + value: ${{ steps.artifacts_url.outputs.url }} +secrets: + +runs: + using: "composite" + steps: + - name: Install mandatory packages + run: | + sudo apt update && sudo apt -y install curl jq libxml2-utils + pip3 -v install tft-cli + shell: bash + + - name: Set artifacts url + id: artifacts_url + run: | + url="https://artifacts.dev.testing-farm.io" + echo "url=$url" >> $GITHUB_OUTPUT + shell: bash + + - name: Create status pull request status name + id: create_status_name + run: echo "name=Plan ${{ inputs.plan }} on ${{ inputs.distro }} ${{ inputs.arch }} for [PR ${{ inputs.pr_number }}](${{ inputs.pr_url }})" >> $GITHUB_OUTPUT + shell: bash + + - name: Schedule a test on Testing Farm + id: sched_test + run: | + cat << EOF > request.json + { + "api_key": "${{ inputs.testing_farm_token }}", + "test": { + "fmf": { + "url": "https://github.com/${{ inputs.repo }}.git", + "ref": "${{ inputs.branch }}", + "name": "${{ inputs.plan }}" + } + }, + "environments": [{ + "arch": "${{ inputs.arch }}", + "os": { + "compose": "${{ inputs.compose }}" + }, + "tmt": { + "context": { + "pr_id": "${{ inputs.pr_number }}", + "distro": "${{ inputs.distro }}" + } + } + }] + } + EOF + + # DEBUG + jq < request.json + + curl ${{ inputs.api_url }}/requests \ + --data @request.json \ + --header "Content-Type: application/json" \ + --output response.json + + # DEBUG + jq < response.json + + req_id=$(jq -r .id response.json) + echo "req_id=$req_id" >> $GITHUB_OUTPUT + shell: bash + + - name: Switch pull request state to running + id: running + run: | + # Create running.json file for query, whether job is finished or not. + cat << EOF > running.json + { + "sha": "${{ inputs.head_sha }}", + "state": "pending", + "context": "Testing Farm - ${{ steps.create_status_name.outputs.name }}", + "description": "Build started", + "target_url": "${{ steps.artifacts_url.outputs.url }}/${{ steps.sched_test.outputs.req_id }}" + } + EOF + + # Update GitHub status description to 'Build started' + curl -X POST \ + -H "Authorization: Bearer ${{ inputs.token }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/${{ inputs.branch }} \ + --data @running.json > update.json + shell: bash + + - name: Check if scheduled test is still running + id: still_running + run: | + CMD=${{ inputs.api_url }}/requests/${{ steps.sched_test.outputs.req_id }} + curl $CMD > job.json + + # DEBUG + jq < job.json + + state=$(jq -r .state job.json) + while [ "$state" == "running" ] || [ "$state" == "new" ] || [ "$state" == "pending" ] || [ "$state" == "queued" ]; do + sleep 30 + curl $CMD > job.json + state=$(jq -r .state job.json) + done + shell: bash + + - name: Get final state of Testing Farm scheduled request + id: final_state + run: | + curl ${{ inputs.api_url }}/requests/${{ steps.sched_test.outputs.req_id }} > job.json + + # DEBUG + jq < job.json + + content=`cat job.json` + echo "result=$content" >> $GITHUB_OUTPUT + echo "not_found=Did not find any plans" >> $GITHUB_OUTPUT + shell: bash + + - name: Process result and ignore if plan is not found + id: check_result + run: | + state=${{ fromJson(steps.final_state.outputs.result).state }} + overall=${{ fromJson(steps.final_state.outputs.result).result.overall }} + can_ignore=${{ contains(fromJson(steps.final_state.outputs.result).result.summary, steps.final_state.outputs.not_found) }} + echo "can_ignore=$can_ignore" + new_state="success" + infra_error="" + log="" + echo "State is $state and result is: $overall" + if [[ "$state" == "complete" ]]; then + if [[ "$overall" != "passed" ]]; then + if [[ "$can_ignore" == "true" ]]; then + echo "The test plan was not found, ignoring error." + new_state="ignored" + else + new_state="failed" + fi + fi + else + # Mark job in case of infrastructure issues. Report to Testing Farm team + infra_error="- Infra problems" + new_state="failed" + log="pipeline.log" + fi + echo "New State is: $new_state" + echo "Infra state is: $infra_error" + echo "FINAL_STATE=$new_state" >> $GITHUB_OUTPUT + echo "INFRA_STATE=$infra_error" >> $GITHUB_OUTPUT + echo "RESULT_SUMMARY=$result_summary" >> $GITHUB_OUTPUT + echo "LOG=$log" >> $GITHUB_OUTPUT + shell: bash + + - name: Switch pull request GitHub status to final state + if: ${{ inputs.update_pull_request_status == 'true' }} + run: | + cat << EOF > final_request.json + { + "sha": "${{ inputs.head_sha }}", + "state": "${{ steps.check_result.outputs.FINAL_STATE }}", + "context": "Testing Farm - ${{ steps.create_status_name.outputs.name }}", + "description": "Build finished ${{ steps.check_result.outputs.INFRA_STATE }}", + "target_url": "${{ steps.artifacts_url.outputs.url }}/${{ steps.sched_test.outputs.req_id }}/${{ steps.check_result.outputs.LOG }}" + } + EOF + # Switch Github status to proper state + curl -X POST -H "Authorization: Bearer ${{ inputs.token }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/${{ inputs.branch }} \ + --data @final_request.json > final_response.json + shell: bash + + - name: Create github summary + shell: bash + run: | + if [ -z "${{ steps.check_result.outputs.INFRA_STATE }}" ]; then + infra_state=OK + else + infra_state=Failed + fi + tfaga_summary_header="Summary for ${{ steps.create_status_name.outputs.name }}:", + if ! cat "$GITHUB_STEP_SUMMARY" | grep "$tfaga_summary_header" > /dev/null; then + echo "$tfaga_summary_header" >> "$GITHUB_STEP_SUMMARY" + echo "| Compose | Arch | Infrastructure State | Test result | Link to logs |" >> "$GITHUB_STEP_SUMMARY" + echo "|---------|------|----------------------|-------------|--------------|" >> "$GITHUB_STEP_SUMMARY" + fi + echo "|${{ inputs.distro }}|${{ inputs.arch }}| $infra_state |"\ + " ${{ steps.check_result.outputs.FINAL_STATE }}|"\ + "${{ steps.artifacts_url.outputs.url }}/${{ steps.sched_test.outputs.req_id }}|" >> "$GITHUB_STEP_SUMMARY" + + - name: Show tests details + shell: bash + run: | + xunit='${{ fromJson(steps.final_state.outputs.result).result.xunit }}' + if [[ ! -z "$xunit" ]]; then + echo "$xunit" > xunit.xml + xmllint --format xunit.xml > results.xml + python3 .github/actions/run-on-testing-farm/src/summary.py results.xml >> $GITHUB_STEP_SUMMARY + fi + + - name: Exit with error in case of failure in test + shell: bash + run: | + final_state="${{ steps.check_result.outputs.FINAL_STATE }}" + if [ "$final_state" == "failed" ]; then + exit 1 + fi + diff --git a/.github/actions/run-on-testing-farm/src/summary.py b/.github/actions/run-on-testing-farm/src/summary.py new file mode 100644 index 00000000..5ff2da2a --- /dev/null +++ b/.github/actions/run-on-testing-farm/src/summary.py @@ -0,0 +1,59 @@ +import argparse +import xml.etree.ElementTree as ET + + +def parse_result(results): + + tree = ET.parse(results) + root = tree.getroot() + + for testsuite in root.findall('testsuite'): + name = testsuite.get('name') + overall = testsuite.get('result') + num_tests = testsuite.get('tests') + formatted_logs = "No logs found" + logs = testsuite.find('logs') + + if logs: + logs_list = map(lambda x: f"[{x[0]}]({x[1]})", + map(lambda l: (l.get('name'), l.get('href')), logs.findall('log'))) + formatted_logs = ", ".join(logs_list) + # Print 1 line table with overall result as the summary + print("|Test suite| Result | Logs |") + print("|----------|--------|------|") + print(f"| {name} | {overall} | {formatted_logs}|") + print(f"
Test details") + print("

\n") + print("| Test | Result | Logs |") + print("|------|--------|------|") + for testcase in testsuite.findall('testcase'): + name = testcase.get('name') + result = testcase.get('result') + formatted_logs = "No logs found" + logs = testcase.find('logs') + if logs: + logs_list = map(lambda x: f"[{x[0]}]({x[1]})", + map(lambda l: (l.get('name'), l.get('href')), logs.findall('log'))) + formatted_logs = ", ".join(logs_list) + print(f"| {name} | {result} | {formatted_logs}|") + print("\n

") + print("
") + + +def main(): + parser = argparse.ArgumentParser( + description="Print results in markdown format") + + parser.add_argument( + 'filename', help="The file containing the xunit xml file returned by Testing Farm") + + args = parser.parse_args() + + if args.filename: + parse_result(args.filename) + else: + print("No results to show") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/pr-test.yaml b/.github/workflows/pr-test.yaml new file mode 100644 index 00000000..86347fea --- /dev/null +++ b/.github/workflows/pr-test.yaml @@ -0,0 +1,57 @@ +name: PR test + +on: + workflow_call: + inputs: + distro: + required: true + type: string + arch: + required: true + type: string + compose: + required: true + type: string + plan: + required: true + type: string + pr_number: + required: true + type: string + pr_url: + required: true + type: string + repo: + required: true + type: string + branch: + required: true + type: string + head_sha: + required: true + type: string + secrets: + token: + required: true + testing_farm_token: + required: true + +jobs: + run-test: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - name: "Plan ${{ inputs.plan }} on ${{ inputs.distro }} ${{ inputs.arch }} for [PR ${{ inputs.pr_number }}](${{ inputs.pr_url }})" + uses: ./.github/actions/run-on-testing-farm + with: + distro: ${{ inputs.distro }} + compose: ${{ inputs.compose }} + arch: ${{ inputs.arch }} + plan: ${{ inputs.plan }} + pr_number: ${{ inputs.pr_number }} + pr_url: ${{ inputs.pr_url }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + head_sha: ${{ inputs.head_sha }} + token: ${{ secrets.token }} + testing_farm_token: ${{ secrets.testing_farm_token }} diff --git a/.github/workflows/register_pr.yaml b/.github/workflows/register_pr.yaml new file mode 100644 index 00000000..83e97b5a --- /dev/null +++ b/.github/workflows/register_pr.yaml @@ -0,0 +1,24 @@ +name: Register PR + +on: + pull_request: + types: [opened, reopened, synchronize] + +jobs: + register: + runs-on: ubuntu-22.04 + - name: Save PR info + run: | + mkdir -p ./pr + echo "{ + \"pr_number\": \"${{ github.event.number }}\", + \"pr_url\": \"${{ github.event.pull_request.issue_url }}\", + \"repo\": \"${{ github.event.pull_request.head.repo.full_name }}\", + \"branch\": \"${{ github.event.pull_request.head.ref }}\", + \"head_sha\": \"${{ github.event.pull_request.head.sha }}\" + }" > ./pr/pr_data.json + - uses: actions/upload-artifact@v3 + with: + name: pr_data + path: pr/ + diff --git a/.github/workflows/tiers.yaml b/.github/workflows/tiers.yaml new file mode 100644 index 00000000..edbdc0b9 --- /dev/null +++ b/.github/workflows/tiers.yaml @@ -0,0 +1,79 @@ +name: Tier tests + +on: + workflow_call: + inputs: + tier: + type: string + pr_number: + required: true + type: string + pr_url: + required: true + type: string + repo: + required: true + type: string + branch: + required: true + type: string + head_sha: + required: true + type: string + secrets: + token: + required: true + testing_farm_token: + required: true + +jobs: + tier1: + runs-on: ubuntu-22.04 + name: 'Tier 1' + steps: + - uses: actions/checkout@v3 + - name: "Tier" + uses: ./.github/actions/run-on-testing-farm + with: + distro: "fedora-37" + compose: "Fedora-37" + arch: "x86_64" + plan: "tier1" + pr_number: ${{ inputs.pr_number }} + pr_url: ${{ inputs.pr_url }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + head_sha: ${{ inputs.head_sha }} + token: ${{ secrets.token }} + testing_farm_token: ${{ secrets.testing_farm_token }} + tier2: + runs-on: ubuntu-22.04 + name: 'Tier 2' + needs: tier1 + if: contains(${{ inputs.tier }}, "tier2") + strategy: + fail-fast: false + matrix: + compose: ["Fedora-37", "Fedora-38"] + arch: ["x86_64"] + steps: + - uses: actions/checkout@v3 + - name: "Resolve distro" + id: resolve_distro + shell: bash + run: | + echo "distro=${{ matrix.compose }}" | tr '[:upper:]' '[:lower:]' >> $GITHUB_OUTPUT + - name: "Plan tier2 on ${{ steps.resolve_distro.outputs.distro }} ${{ matrix.arch }} for [PR ${{ inputs.pr_number }}](${{ inputs.pr_url }})" + uses: ./.github/actions/run-on-testing-farm + with: + distro: ${{ steps.resolve_distro.outputs.distro }} + compose: ${{ matrix.distro }} + arch: ${{ matrix.arch }} + plan: "tier2" + pr_number: ${{ inputs.pr_number }} + pr_url: ${{ inputs.pr_url }} + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + head_sha: ${{ inputs.head_sha }} + token: ${{ secrets.token }} + testing_farm_token: ${{ secrets.testing_farm_token }} diff --git a/.github/workflows/trigger-on-pr.yaml b/.github/workflows/trigger-on-pr.yaml new file mode 100644 index 00000000..690f6236 --- /dev/null +++ b/.github/workflows/trigger-on-pr.yaml @@ -0,0 +1,85 @@ +name: Testing Farm + +on: + workflow_run: + workflows: [Register PR] + types: [completed] + +jobs: + get_data: + outputs: + pr_number: ${{ steps.recover.outputs.pr_number }} + pr_url: ${{ steps.recover.outputs.pr_url}} + repo: ${{ steps.recover.outputs.repo }} + branch: ${{ steps.recover.outputs.branch }} + head_sha: ${{ steps.recover.outputs.head_sha }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - name: 'Download artifact' + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "pr_data" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); + - name: 'Unzip' + id: unzip + shell: bash + run: | + unzip pr_data.zip + echo "data=\"$(cat pr/pr_data.json)\"" >> $GITHUB_OUTPUT + - name: 'Recover data' + id: recover + shell: bash + run: | + echo "pr_number=${{ fromJson(steps.unzip.outputs.data).pr_number }}" >> $GITHUB_OUTPUT + echo "pr_url=${{ fromJson(steps.unzip.outputs.data).pr_url }}" >> $GITHUB_OUTPUT + echo "repo=${{ fromJson(steps.unzip.outputs.data).repo }}" >> $GITHUB_OUTPUT + echo "branch=${{ fromJson(steps.unzip.outputs.data).branch }}" >> $GITHUB_OUTPUT + echo "head_sha=${{ fromJson(steps.unzip.outputs.data).head_sha }}" >> $GITHUB_OUTPUT + pr_test: + needs: get_data + name: PR specific test + uses: ./.github/workflows/pr-test.yaml + with: + distro: "fedora-37" + compose: "Fedora-37" + arch: "x86_64" + plan: ${{ needs.get_data.outputs.pr_number }} + pr_number: ${{ needs.get_data.outputs.pr_number }} + pr_url: ${{ needs.get_data.outputs.pr_url }} + repo: ${{ needs.get_data.outputs.repo }} + branch: ${{ needs.get_data.outputs.branch }} + head_sha: ${{ needs.get_data.outputs.head_sha }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + testing_farm_token: ${{ secrets.TESTING_FARM_API_TOKEN }} + tier: + needs: [get_data, pr_test] + name: Tier tests + uses: ./.github/workflows/tiers.yaml + with: + tier: "tier2" + pr_number: ${{ needs.get_data.outputs.pr_number }} + pr_url: ${{ needs.get_data.outputs.pr_url }} + repo: ${{ needs.get_data.outputs.repo }} + branch: ${{ needs.get_data.outputs.branch }} + head_sha: ${{ needs.get_data.outputs.head_sha }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + testing_farm_token: ${{ secrets.TESTING_FARM_API_TOKEN }} + diff --git a/.packit.yaml b/.packit.yaml deleted file mode 100644 index 2fb5a38b..00000000 --- a/.packit.yaml +++ /dev/null @@ -1,12 +0,0 @@ -jobs: -- job: tests - trigger: pull_request - targets: - - fedora-stable - - centos-stream-9-x86_64 - skip_build: true - tf_extra_params: - environments: - - tmt: - context: - target_PR_branch: "main" diff --git a/plans/tier1.fmf b/plans/tier1.fmf new file mode 100644 index 00000000..bd20327f --- /dev/null +++ b/plans/tier1.fmf @@ -0,0 +1,49 @@ +summary: + Tests for keylime rust agent + +environment+: + TPM_BINARY_MEASUREMENTS: /var/tmp/binary_bios_measurements + KEYLIME_RUST_CODE_COVERAGE: 0 + RPM_AGENT_COVERAGE: 0 + +context+: + agent: rust + +prepare: + - how: shell + script: + - systemctl disable --now dnf-makecache.service || true + - systemctl disable --now dnf-makecache.timer || true + +adjust: + - when: distro == centos-stream-9 + prepare+: + - how: shell + script: + - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + - yum config-manager --set-enabled crb + - when: distro == centos-stream-8 + enabled: false + # disable temporarily on Rawhide since latest Rust agent cannot be compiled + - when: distro == fedora-rawhide + enabled: false + + - when: "distro == fedora-36 or distro == fedora-37" + prepare+: + - how: shell + order: 99 + script: + - yum -y downgrade tpm2-tss + +discover: + how: fmf + test: + - /setup/configure_tpm_emulator + - /setup/install_upstream_keylime + - /setup/install_upstream_rust_keylime + - /setup/enable_keylime_debug_messages + - /setup/configure_kernel_ima_module/ima_policy_signing + - /functional/basic-attestation-on-localhost + +execute: + how: tmt diff --git a/plans/tier2.fmf b/plans/tier2.fmf new file mode 100644 index 00000000..e8a5e1dc --- /dev/null +++ b/plans/tier2.fmf @@ -0,0 +1,58 @@ +summary: + Tests for keylime rust agent + +environment+: + TPM_BINARY_MEASUREMENTS: /var/tmp/binary_bios_measurements + KEYLIME_RUST_CODE_COVERAGE: 1 + RPM_AGENT_COVERAGE: 0 + +context+: + agent: rust + +prepare: + - how: shell + script: + - systemctl disable --now dnf-makecache.service || true + - systemctl disable --now dnf-makecache.timer || true + +adjust: + - when: distro == centos-stream-9 + prepare+: + - how: shell + script: + - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + - yum config-manager --set-enabled crb + - when: distro == centos-stream-8 + enabled: false + # disable temporarily on Rawhide since latest Rust agent cannot be compiled + - when: distro == fedora-rawhide + enabled: false + + - when: "distro == fedora-36 or distro == fedora-37" + prepare+: + - how: shell + order: 99 + script: + - yum -y downgrade tpm2-tss + # disable code coverage measurement everywhere except F37 and CS9 + - when: distro != centos-stream-9 and distro != fedora-37 + environment+: + KEYLIME_RUST_CODE_COVERAGE: 0 + discover+: + test-: + - /setup/generate_usptream_rust_keylime_code_coverage + +discover: + how: fmf + test: + - /setup/configure_tpm_emulator + - /setup/install_upstream_keylime + - /setup/install_upstream_rust_keylime + - /setup/enable_keylime_debug_messages + - /setup/configure_kernel_ima_module/ima_policy_signing + - "/functional/.*" + - /upstream/run_rust_keylime_tests + - /setup/generate_usptream_rust_keylime_code_coverage + +execute: + how: tmt