diff --git a/.github/actions/check-chart-locks/action.yml b/.github/actions/check-chart-locks/action.yml new file mode 100644 index 0000000000..be13eeafae --- /dev/null +++ b/.github/actions/check-chart-locks/action.yml @@ -0,0 +1,63 @@ +name: Check For Lock +description: | + Checks for the existence of chart-name in a given chart lock file. If a chart + is found to be locked at the URL searched, the lockpath is returned should the + caller need it for further evaluation. + + Designed to work with OWNERS file merges to prevent chart naming conflicts. +inputs: + # e.g. vault + chart-name: + required: true + description: + Check the lock status of this chart by name. + fail-workflow-if-locked: + required: false + default: 'false' + description: | + Forces a failure of this action when the chart is locked. Must explicitly + be set to the value 'true'. All other values (even a boolean true) are + considered false. +outputs: + # e.g. true/false + chart-is-locked: + description: Whether the chart provided via the chart-name input is locked + value: ${{ steps.set-lock-state.outputs.chart-is-locked }} + # e.g. charts/category/vendor/chartname if package-is-locked is true, else null + locked-to-path: + description: The path to which a locked chart is locked. + value: ${{ steps.check-for-chart-lock.outputs.locked-to-path }} +runs: + using: composite + steps: + - name: Generate Locks + id: generate-locks + uses: ./.github/actions/generate-chart-locks + - name: Check lockfile for chart lock + id: check-for-chart-lock + shell: bash + run: | + set -e + echo "Ensuring expected key exists in lock JSON." + jq --exit-status .packages < ${{ steps.generate-locks.outputs.lockfile-path }} + + echo "Checking if chart '${{ inputs.chart-name }}' is locked." + LOCK_PATH=$(jq -r '.packages."${{ inputs.chart-name }}"' < ${{ steps.generate-locks.outputs.lockfile-path }}) + + echo "locked-to-path=${LOCK_PATH}" | tee -a $GITHUB_OUTPUT + + # Defaults to a locked state as a safeguard. + - name: Set lock state output + shell: bash + id: set-lock-state + run: | + echo "chart-is-locked=${{ steps.check-for-chart-lock.outputs.locked-to-path != 'null' }}" | tee -a $GITHUB_OUTPUT + + - name: Fail if requested and the chart is locked + shell: bash + if: | + inputs.fail-workflow-if-locked == 'true' && + steps.set-lock-state.outputs.chart-is-locked == 'true' + run: | + echo "::error::Workflow is failing at the caller's request." + exit -1 \ No newline at end of file diff --git a/.github/actions/generate-chart-locks/action.yaml b/.github/actions/generate-chart-locks/action.yaml new file mode 100644 index 0000000000..4865bfe1f4 --- /dev/null +++ b/.github/actions/generate-chart-locks/action.yaml @@ -0,0 +1,78 @@ +name: Generate Chart Locks +description: | + Generates chart locks at runtime and places the generated contents on the + filesystem. + + For pull_request / pull_request_target events, this will pull down the branch + receiving the changes as the source of truth. + + In the production repository, this will always pull the main branch, regardless + of which branch is receiving the pull request. + + It is expected that the CI scripts have already been installed at call time. +inputs: + to-file: + description: Where to write the chart-locks.json file. + default: "/tmp/chart-locks.json" + required: false + generator-cmd-path: + description: | + The path to the generate-chart-locks command. This action expects CI + scripts to be installed by the caller, and so it stands to reason the + caller may install scripts at various locations. + default: "ve1/bin/generate-chart-locks" + required: false +outputs: + lockfile-path: + description: | + Where the lock file was written. Mostly placed as an output to simplify + workflows for callers. + value: ${{ steps.generate-chart-locks.outputs.lockfile-path }} +runs: + using: composite + steps: + - name: Resolve repository ref + id: resolve + shell: bash + run: | + set -e + + # PRs to the production repository will always use main. + if [ "${GITHUB_REPOSITORY}" == "openshift-helm-charts/charts" ]; then + echo "Running in the production repository." + echo "The only allowed ref is 'refs/heads/main'." + echo "ref=refs/heads/main" | tee -a $GITHUB_OUTPUT + exit 0 + fi + + echo "GITHUB_EVENT_NAME = '${GITHUB_EVENT_NAME}'" + echo "GITHUB_BASE_REF = '${GITHUB_BASE_REF}'" + + # GITHUB_BASE_REF is set for pull_requests/pull_requests_targets, but empty for + # workflow_dispatch. We'll set it to main if it's not set. + resolvedRef="refs/heads/${GITHUB_BASE_REF:-"main"}" + echo "ref=${resolvedRef}" | tee -a $GITHUB_OUTPUT + - name: Checkout + id: clone-repository + uses: actions/checkout@v3 + with: + ref: ${{ steps.resolve.outputs.ref }} + path: temp-gen-chart-lock-repo + - name: Setting up python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Generate lock file JSON from existing charts + working-directory: temp-gen-chart-lock-repo + id: generate-chart-locks + shell: bash + run: | + set -o pipefail + ${{ inputs.generator-cmd-path }} | jq | tee ${{ inputs.to-file }} + echo "lockfile-path=$(realpath ${{ inputs.to-file }})" | tee -a $GITHUB_OUTPUT + - name: Cleanup + id: cleanup + if: always() + shell: bash + run: | + rm -rf temp-gen-chart-lock-repo diff --git a/.github/actions/get-ocp-range/action.yaml b/.github/actions/get-ocp-range/action.yaml index 1d75e6a776..33a5f31fb0 100644 --- a/.github/actions/get-ocp-range/action.yaml +++ b/.github/actions/get-ocp-range/action.yaml @@ -12,7 +12,7 @@ runs: using: "composite" steps: - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '>=1.20' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 079a7df367..eea038fbc4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,10 +19,10 @@ jobs: github.actor != 'redhat-mercury-bot' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.x Part 1 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -113,18 +113,18 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Checkout PR Branch if: ${{ needs.setup.outputs.run_build == 'true' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} path: "pr-branch" - name: Set up Python 3.x Part 1 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -149,7 +149,7 @@ jobs: ./ve1/bin/check-pr-content --index-branch=${INDEX_BRANCH} --repository=${{ github.repository }} --api-url=${{ github.event.pull_request._links.self.href }} - name: Add 'content-ok' label - uses: actions/github-script@v6 + uses: actions/github-script@v7 if: ${{ steps.check_pr_content.outcome == 'success'}} continue-on-error: true with: @@ -163,7 +163,7 @@ jobs: }) - name: Remove 'content-ok' label - uses: actions/github-script@v6 + uses: actions/github-script@v7 if: ${{ steps.check_pr_content.outcome == 'failure' && contains( github.event.pull_request.labels.*.name, 'content-ok') }} continue-on-error: true with: @@ -183,7 +183,7 @@ jobs: exit 1 - name: Remove 'authorized-request' label from PR - uses: actions/github-script@v6 + uses: actions/github-script@v7 if: ${{ needs.setup.outputs.run_build == 'true' && contains( github.event.pull_request.labels.*.name, 'authorized-request') }} continue-on-error: true with: @@ -266,7 +266,7 @@ jobs: - name: Get profile version set in report provided by the user id: get-profile-version if: ${{ needs.setup.outputs.run_build == 'true' && steps.verify_requires.outputs.report_provided == 'true' }} - uses: mikefarah/yq@v4.35.1 + uses: mikefarah/yq@v4 with: cmd: yq '.metadata.tool.profile.version' ${{ format('./pr-branch/{0}', steps.verify_requires.outputs.provided_report_relative_path) }} @@ -274,7 +274,7 @@ jobs: id: get-kube-range if: ${{ needs.setup.outputs.run_build == 'true' && steps.verify_requires.outputs.report_provided == 'true' }} continue-on-error: true - uses: mikefarah/yq@v4.35.1 + uses: mikefarah/yq@v4 with: cmd: yq '.metadata.chart.kubeversion' ${{ format('./pr-branch/{0}', steps.verify_requires.outputs.provided_report_relative_path) }} @@ -352,7 +352,7 @@ jobs: - name: Comment on PR if: ${{ always() && needs.setup.outputs.run_build == 'true' }} - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -368,7 +368,7 @@ jobs: - name: Add 'authorized-request' label to PR if: ${{ always() && steps.check_pr_content.outcome == 'success' && steps.run-verifier.outcome != 'failure' && needs.setup.outputs.run_build == 'true' }} - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -397,7 +397,7 @@ jobs: - name: Merge PR id: merge_pr if: ${{ steps.approve_pr.conclusion == 'success' }} - uses: pascalgn/automerge-action@v0.15.6 + uses: pascalgn/automerge-action@v0.16.2 env: GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} MERGE_METHOD: squash @@ -418,18 +418,18 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Checkout PR Branch if: ${{ needs.setup.outputs.run_build == 'true' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} path: "pr-branch" - name: Set up Python 3.x Part 1 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -577,7 +577,7 @@ jobs: - name: Alert Slack helm_dev on failure to update metrics continue-on-error: true if: steps.add_metrics.outcome == 'failure' - uses: archive/github-actions-slack@v2.7.0 + uses: archive/github-actions-slack@v2.8.0 with: slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} slack-channel: C02979BDUPL diff --git a/.github/workflows/check-contributor.yml b/.github/workflows/check-contributor.yml index b4b7d2b78c..fff3b1bca3 100644 --- a/.github/workflows/check-contributor.yml +++ b/.github/workflows/check-contributor.yml @@ -40,10 +40,10 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout repository base - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/mercury_bot.yml b/.github/workflows/mercury_bot.yml index 2a7cf50897..338c19c95b 100644 --- a/.github/workflows/mercury_bot.yml +++ b/.github/workflows/mercury_bot.yml @@ -11,10 +11,10 @@ jobs: if: github.event.pull_request.draft == false && github.actor == 'redhat-mercury-bot' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.x Part 1 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -63,7 +63,7 @@ jobs: - name: Comment on PR if: ${{ steps.check_for_owners.outputs.merge_pr == 'false' }} - uses: actions/github-script@v3 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -93,7 +93,7 @@ jobs: - name: Merge PR id: merge_pr if: ${{ steps.approve_pr.conclusion == 'success' }} - uses: pascalgn/automerge-action@v0.15.6 + uses: pascalgn/automerge-action@v0.16.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MERGE_METHOD: squash diff --git a/.github/workflows/owners.yml b/.github/workflows/owners.yml index 9ce895bdd6..e8680070e3 100644 --- a/.github/workflows/owners.yml +++ b/.github/workflows/owners.yml @@ -14,7 +14,7 @@ jobs: SEGMENT_TEST_WRITE_KEY: ${{ secrets.SEGMENT_TEST_WRITE_KEY }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.x Part 1 uses: actions/setup-python@4 diff --git a/.github/workflows/python-style.yml b/.github/workflows/python-style.yml index bec86bdf00..78e9cd3f48 100644 --- a/.github/workflows/python-style.yml +++ b/.github/workflows/python-style.yml @@ -13,7 +13,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python 3.x Part 1 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install style tooling diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d52c49bb21..705389e669 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,7 @@ jobs: needs.check-contributor.outputs.is-repo-owner == 'true' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.sha }} @@ -48,7 +48,7 @@ jobs: fetch-depth: 0 - name: Set up Python 3.x Part 1 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -154,7 +154,7 @@ jobs: - name: Merge PR id: merge_pr if: ${{ steps.check_if_release_pr.outputs.charts_release_branch == 'true' }} - uses: pascalgn/automerge-action@v0.15.6 + uses: pascalgn/automerge-action@v0.16.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MERGE_METHOD: squash diff --git a/.github/workflows/version_check.yml b/.github/workflows/version_check.yml index 9a02da63e0..5f12cdd687 100644 --- a/.github/workflows/version_check.yml +++ b/.github/workflows/version_check.yml @@ -83,7 +83,7 @@ jobs: - name: Checkout software-version branch if: steps.check_repo.outputs.check-version == 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: "software-version" repository: ${{ github.repository }} @@ -150,7 +150,7 @@ jobs: - name: Checkout main branch if: | steps.check_test.outputs.run_tests == 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: "main" token: ${{ secrets.BOT_TOKEN }} @@ -159,7 +159,7 @@ jobs: - name: Set up Python 3.x Part 1 if: | steps.check_test.outputs.run_tests == 'true' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -220,7 +220,7 @@ jobs: - name: Send message to helm_dev slack channel id: notify_dev if: ${{ always() && github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' && steps.run-schedule-tests.conclusion != 'success' }} - uses: archive/github-actions-slack@v2.7.0 + uses: archive/github-actions-slack@v2.8.0 with: slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} slack-channel: C02979BDUPL @@ -233,7 +233,7 @@ jobs: - name: Send message to helm_notify slack channel id: notify if: ${{ always() && github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' && steps.run-schedule-tests.conclusion == 'success' }} - uses: archive/github-actions-slack@v2.7.0 + uses: archive/github-actions-slack@v2.8.0 with: slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} slack-channel: C04K1ARMH8A @@ -288,7 +288,7 @@ jobs: - name: Checkout software-version branch if: steps.check_repo.outputs.check-version == 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: "software-version" repository: ${{ github.repository }} @@ -357,7 +357,7 @@ jobs: - name: Checkout charts main branch if: | steps.check_test.outputs.run_tests == 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: "main" token: ${{ secrets.BOT_TOKEN }} @@ -366,7 +366,7 @@ jobs: - name: Set up Python 3.x Part 1 if: | steps.check_test.outputs.run_tests == 'true' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -429,7 +429,7 @@ jobs: - name: Send message to helm_dev slack channel id: notify_dev if: ${{ always() && github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' && steps.run-schedule-tests.conclusion != 'success' }} - uses: archive/github-actions-slack@v2.7.0 + uses: archive/github-actions-slack@v2.8.0 with: slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} slack-channel: C02979BDUPL @@ -442,7 +442,7 @@ jobs: - name: Send message to helm_notify slack channel id: notify if: ${{ always() && github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' && steps.run-schedule-tests.conclusion == 'success' }} - uses: archive/github-actions-slack@v2.7.0 + uses: archive/github-actions-slack@v2.8.0 with: slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} slack-channel: C04K1ARMH8A diff --git a/scripts/ruff.toml b/scripts/ruff.toml index af87f9db59..054088c47a 100644 --- a/scripts/ruff.toml +++ b/scripts/ruff.toml @@ -1,4 +1,7 @@ ignore = [ "E402", # import ordering (komish): import ordering isn't handled by Black so we need to handle this manually. "E501", # line length (komish): line length is not enforced by Black so we need to handle these manually. -] \ No newline at end of file +] + +# Enable the isort rules. +extend-select = ["I"] diff --git a/scripts/setup.cfg b/scripts/setup.cfg index ebc5ae9d18..0fc80aefcf 100644 --- a/scripts/setup.cfg +++ b/scripts/setup.cfg @@ -47,4 +47,6 @@ console_scripts = pushowners=metrics.pushowners:main update-index=updateindex.updateindex:main user-is-repo-owner=owners.user_is_repo_owner:main - + generate-chart-locks=packagemapping.generatelocks:main + extract-metadata-from-pr=pullrequest.metadata:main + assert-redhat-owners-file-meta=owners.redhat_metadata:main diff --git a/scripts/src/chartprreview/chartprreview.py b/scripts/src/chartprreview/chartprreview.py index 4d0f7d5dbc..7cf0e28c19 100644 --- a/scripts/src/chartprreview/chartprreview.py +++ b/scripts/src/chartprreview/chartprreview.py @@ -1,17 +1,16 @@ -import re +import argparse +import hashlib import os import os.path -import sys -import argparse +import re import subprocess -import hashlib - -from environs import Env +import sys -import semver -import semantic_version import requests +import semantic_version +import semver import yaml +from environs import Env try: from yaml import CLoader as Loader @@ -19,11 +18,10 @@ from yaml import Loader sys.path.append("../") -from report import report_info -from report import verifier_report -from signedchart import signedchart from pullrequest import prartifact from reporegex import matchers +from report import report_info, verifier_report +from signedchart import signedchart from tools import gitutils diff --git a/scripts/src/chartprreview/chartprreview_test.py b/scripts/src/chartprreview/chartprreview_test.py index e641aa5565..c306164055 100644 --- a/scripts/src/chartprreview/chartprreview_test.py +++ b/scripts/src/chartprreview/chartprreview_test.py @@ -1,8 +1,12 @@ import os + import pytest -from chartprreview.chartprreview import verify_user -from chartprreview.chartprreview import check_owners_file_against_directory_structure -from chartprreview.chartprreview import write_error_log + +from chartprreview.chartprreview import ( + check_owners_file_against_directory_structure, + verify_user, + write_error_log, +) def test_verify_user(): diff --git a/scripts/src/chartrepomanager/chartrepomanager.py b/scripts/src/chartrepomanager/chartrepomanager.py index 02c9802d33..65af0fbe1d 100644 --- a/scripts/src/chartrepomanager/chartrepomanager.py +++ b/scripts/src/chartrepomanager/chartrepomanager.py @@ -22,31 +22,33 @@ import argparse import base64 import json -import shutil import os -import sys import re +import shutil import subprocess +import sys import tempfile import time import urllib.parse -from environs import Env import yaml +from environs import Env try: - from yaml import CLoader as Loader, CDumper as Dumper + from yaml import CDumper as Dumper + from yaml import CSafeLoader as SafeLoader except ImportError: - from yaml import Loader, Dumper + from yaml import Dumper, SafeLoader sys.path.append("../") -from report import report_info -from chartrepomanager import indexannotations -from signedchart import signedchart from pullrequest import prartifact from reporegex import matchers +from report import report_info +from signedchart import signedchart from tools import gitutils +from chartrepomanager import indexannotations + def _encode_chart_entry(chart_entry): """Encode the chart_entry to base64. This is needed to pass it as an argument to @@ -271,7 +273,7 @@ def create_index_from_chart(chart_file_name): p = out.stdout.decode("utf-8") print(p) print(out.stderr.decode("utf-8")) - crt = yaml.load(p, Loader=Loader) + crt = yaml.load(p, Loader=SafeLoader) return crt @@ -374,7 +376,7 @@ def update_chart_annotation( data = open( os.path.join("charts", category, organization, chart, "OWNERS") ).read() - out = yaml.load(data, Loader=Loader) + out = yaml.load(data, Loader=SafeLoader) vendor_name = out["vendor"]["name"] annotations["charts.openshift.io/provider"] = vendor_name @@ -392,7 +394,7 @@ def update_chart_annotation( print(out.stderr.decode("utf-8")) fd = open(os.path.join(dr, chart, "Chart.yaml")) - data = yaml.load(fd, Loader=Loader) + data = yaml.load(fd, Loader=SafeLoader) if "annotations" not in data: data["annotations"] = annotations diff --git a/scripts/src/chartrepomanager/indexannotations.py b/scripts/src/chartrepomanager/indexannotations.py index 9533be960f..192e315f78 100644 --- a/scripts/src/chartrepomanager/indexannotations.py +++ b/scripts/src/chartrepomanager/indexannotations.py @@ -1,4 +1,5 @@ import sys + import semantic_version sys.path.append("../") @@ -30,21 +31,21 @@ def getIndexAnnotations(ocp_version_range, report_path): full_version = annotations[annotation] if full_version != "N/A" and semantic_version.validate(full_version): ver = semantic_version.Version(full_version) - set_annotations[ - "charts.openshift.io/testedOpenShiftVersion" - ] = f"{ver.major}.{ver.minor}" + set_annotations["charts.openshift.io/testedOpenShiftVersion"] = ( + f"{ver.major}.{ver.minor}" + ) else: - set_annotations[ - "charts.openshift.io/testedOpenShiftVersion" - ] = annotations[annotation] + set_annotations["charts.openshift.io/testedOpenShiftVersion"] = ( + annotations[annotation] + ) else: if annotation == "charts.openshift.io/supportedOpenShiftVersions": OCPSupportedSet = True set_annotations[annotation] = annotations[annotation] if not OCPSupportedSet: - set_annotations[ - "charts.openshift.io/supportedOpenShiftVersions" - ] = ocp_version_range + set_annotations["charts.openshift.io/supportedOpenShiftVersions"] = ( + ocp_version_range + ) return set_annotations diff --git a/scripts/src/checkautomerge/checkautomerge.py b/scripts/src/checkautomerge/checkautomerge.py index 25f61b91cb..0d7b5fd3fe 100644 --- a/scripts/src/checkautomerge/checkautomerge.py +++ b/scripts/src/checkautomerge/checkautomerge.py @@ -1,7 +1,7 @@ -import time -import sys import argparse import os +import sys +import time import requests diff --git a/scripts/src/checkprcontent/checkpr.py b/scripts/src/checkprcontent/checkpr.py index 4ecc69bbf0..01c85b1a51 100644 --- a/scripts/src/checkprcontent/checkpr.py +++ b/scripts/src/checkprcontent/checkpr.py @@ -1,13 +1,12 @@ -import re -import os -import sys import argparse import json +import os +import re +import sys import requests import semver import yaml - from reporegex import matchers try: @@ -17,8 +16,8 @@ sys.path.append("../") from owners import owners_file -from report import verifier_report from pullrequest import prartifact +from report import verifier_report from tools import gitutils ALLOW_CI_CHANGES = "allow/ci-changes" diff --git a/scripts/src/indexfile/index.py b/scripts/src/indexfile/index.py index b45201ddde..51a83791b7 100644 --- a/scripts/src/indexfile/index.py +++ b/scripts/src/indexfile/index.py @@ -1,8 +1,9 @@ import json +import sys + import requests -import yaml import semantic_version -import sys +import yaml sys.path.append("../") diff --git a/scripts/src/metrics/metrics.py b/scripts/src/metrics/metrics.py index 00e96aa834..d3b43fa2d9 100644 --- a/scripts/src/metrics/metrics.py +++ b/scripts/src/metrics/metrics.py @@ -1,16 +1,18 @@ import argparse import itertools -import requests -import sys -import analytics import os import re +import sys + +import analytics +import requests from github import Github sys.path.append("../") +from collections import OrderedDict + from indexfile import index from pullrequest import prepare_pr_comment as pr_comment -from collections import OrderedDict from reporegex import matchers file_pattern = re.compile( diff --git a/scripts/src/metrics/pushowners.py b/scripts/src/metrics/pushowners.py index 3612ac9764..d1d8b2937b 100644 --- a/scripts/src/metrics/pushowners.py +++ b/scripts/src/metrics/pushowners.py @@ -1,5 +1,6 @@ import argparse import sys + import analytics sys.path.append("../") diff --git a/scripts/src/owners/checkuser.py b/scripts/src/owners/checkuser.py index 05b615b375..be1f12d039 100644 --- a/scripts/src/owners/checkuser.py +++ b/scripts/src/owners/checkuser.py @@ -10,10 +10,11 @@ exit code 1 if pull request contains restricted files and user is not authorized to modify them. """ -import re import argparse import os +import re import sys + import yaml try: @@ -24,7 +25,6 @@ sys.path.append("../") from pullrequest import prartifact - OWNERS_FILE = "OWNERS" VERSION_FILE = "release/release_info.json" THIS_FILE = "scripts/src/owners/checkuser.py" diff --git a/scripts/src/owners/owners_file.py b/scripts/src/owners/owners_file.py index 7cbeac1aab..f8913cd1e4 100644 --- a/scripts/src/owners/owners_file.py +++ b/scripts/src/owners/owners_file.py @@ -3,9 +3,9 @@ import yaml try: - from yaml import CLoader as Loader + from yaml import CSafeoader as SafeLoader except ImportError: - from yaml import Loader + from yaml import SafeLoader def get_owner_data(category, organization, chart): @@ -17,7 +17,7 @@ def get_owner_data(category, organization, chart): def get_owner_data_from_file(owner_path): try: with open(owner_path) as owner_data: - owner_content = yaml.load(owner_data, Loader=Loader) + owner_content = yaml.load(owner_data, Loader=SafeLoader) return True, owner_content except Exception as err: print(f"Exception loading OWNERS file: {err}") @@ -33,6 +33,15 @@ def get_vendor(owner_data): return vendor +def get_vendor_label(owner_data): + vendor = "" + try: + vendor = owner_data["vendor"]["label"] + except Exception: + pass + return vendor + + def get_chart(owner_data): chart = "" try: diff --git a/scripts/src/owners/redhat_metadata.py b/scripts/src/owners/redhat_metadata.py new file mode 100644 index 0000000000..65687c15ea --- /dev/null +++ b/scripts/src/owners/redhat_metadata.py @@ -0,0 +1,95 @@ +from argparse import ArgumentParser + +from owners import owners_file + +REQUIRED_VENDOR_LABEL = "redhat" +REQUIRED_VENDOR_NAME = "Red Hat" +REQUIRED_CHART_PREFIX = "redhat-" + + +class RedHatOwnersFileInvalidContentsError(Exception): + message = "The OWNERS does not contain criteria required for Red Hatters" + + def __init__(self, message): + if len(message) > 0: + self.message = message + + super().__init__(self.message) + + +def assert_redhat_metadata(contents): + """Ensures contents of owners file for Red Hat submissions + + Red Hat submissions are expected to have certain metadata. + + - The owner label must be "redhat" + - The owner organizaton name must be "Red Hat" + - The chart's name must be prefixed with "redhat-" + + This raises an exception if the metadata is incorrect. + + Args: + contents: The contents of the owners + file to be evaluated. This should have already + been read from disk and yaml.loaded + + Raises: + RedHatOwnersFileInvalidContentsError: In cases where the + content is not as expected. + + Returns: + Nothing. + """ + + if not owners_file.get_chart(contents).startswith(REQUIRED_CHART_PREFIX): + raise RedHatOwnersFileInvalidContentsError( + f"The chart name must be prefixed with '{REQUIRED_CHART_PREFIX}'." + ) + + if owners_file.get_vendor_label(contents) != REQUIRED_VENDOR_LABEL: + raise RedHatOwnersFileInvalidContentsError( + f"OWNERS file did not have expected vendor label: '{REQUIRED_VENDOR_LABEL}'" + ) + + if owners_file.get_vendor(contents) != REQUIRED_VENDOR_NAME: + raise RedHatOwnersFileInvalidContentsError( + f"OWNERS file did not have expected vendor name: '{REQUIRED_VENDOR_NAME}'" + ) + + +def main(): + """The CLI entrypoint. + + Return codes: + 0: everything has gone well. + 10: The OWNERS file was not found. + 20: The OWNERS file has invalid content. + """ + parser = ArgumentParser() + parser.add_argument("category") + parser.add_argument("organization") + parser.add_argument("chartname") + args = parser.parse_args() + print( + f"[Info] Checking OWNERS file content for {args.category}/{args.organization}/{args.chartname}" + ) + + ownerDataFound, ownerdata = owners_file.get_owner_data( + args.category, args.organization, args.chartname + ) + if not ownerDataFound: + print("[Error] OWNERS file not found") + return 10 + + try: + assert_redhat_metadata(ownerdata) + except RedHatOwnersFileInvalidContentsError as e: + print(f"[Error] Invalid contents, {e}") + return 20 + + print("[Info] LGTM!") + return 0 + + +if __name__ == "__main__": + main() diff --git a/scripts/src/owners/user_is_repo_owner.py b/scripts/src/owners/user_is_repo_owner.py index ed2e532f7b..731c2cf2c6 100644 --- a/scripts/src/owners/user_is_repo_owner.py +++ b/scripts/src/owners/user_is_repo_owner.py @@ -11,6 +11,7 @@ import os import sys + import yaml try: diff --git a/tests/functional/__init__.py b/scripts/src/packagemapping/__init__.py similarity index 100% rename from tests/functional/__init__.py rename to scripts/src/packagemapping/__init__.py diff --git a/scripts/src/packagemapping/generatelocks.py b/scripts/src/packagemapping/generatelocks.py new file mode 100644 index 0000000000..fbfcda5f2c --- /dev/null +++ b/scripts/src/packagemapping/generatelocks.py @@ -0,0 +1,124 @@ +import re +import sys +from datetime import datetime, timezone +from glob import glob +from json import dumps as to_json + +from owners import owners_file + + +def ownerfile_regex(): + """Return a regex statement that parses relative filepaths of an OWNERS file. + + The filepath is expected to be from the charts directory forward. + + Returns: + The compiled regex that groups the category, organization, and chart name. + + E.g. category | organization | chartname + partner | hashicorp | vault + """ + + pattern = re.compile(r"charts/(partners|redhat|community)/([\w-]+)/([\w-]+)/OWNERS") + return pattern + + +def logError(msg, file=sys.stderr): + """logError just prints the msg with an ERROR caption to stderr unless otherwise defined""" + print(f"[ERROR] {msg}", file=file) + + +def logInfo(msg, file=sys.stderr): + """logError just prints the msg with an INFO caption to stderr unless otherwise defined""" + print(f"[INFO] {msg}", file=file) + + +def logWarn(msg, file=sys.stderr): + """logWarn just prints the msg with an WARN caption to stderr unless otherwise defined""" + print(f"[WARN] {msg}", file=file) + + +def main(): + """Generates a mapping of chart names to their associated paths. + + Prints the resulting output as a JSON blob. + + Return codes: + 0: All is well. + 10: Parsing failure for the input path to a given OWNERS file. + 20: One of the expected directory values is empty. + 30: The OWNERS file at the provided path did not load correctly. + 35: The OWNERS file content and the directory structure are mismatched. + 40: A duplicate chart name entry has been found. + 50: The resulting data contained no entries, which + is certainly unexpected. + """ + packages = {} + for filename in glob("charts/**/OWNERS", recursive=True): + logInfo(f"processing file {filename}") + pattern = ownerfile_regex() + matched = pattern.match(filename) + if matched is None: + logError( + f"did not successfully parse the input. Did you run this in the right place? filename: {filename}", + ) + return 10 + + category, organization, chart = matched.groups() + if category is None or organization is None or chart is None: + logError( + f"did not successfully parse the input filename: {filename}", + "expecting format charts/{category}/{organization}/{chartname}/OWNERS. Did you run this in the right place?", + ) + return 20 + + ownersContentLoaded, ownersContent = owners_file.get_owners_data_from_file( + filename + ) + if not ownersContentLoaded: + logError(f"Failed to load OWNERS file content. filename: {filename}") + return 30 + + owners_value_chart_name = owners_file.get_chart(ownersContent) + owners_value_vendor_label = owners_file.get_vendor_label(ownersContent) + + if owners_value_chart_name != chart: + logError( + f"the chart name in the OWNERS file did not match the chart name directory structure. OWNERS_FILE_VALUE={owners_value_chart_name}, DIRECTORY_VALUE:{chart}" + ) + return 35 + + if owners_value_vendor_label != organization: + logError( + f"the vendor label in the OWNERS file did not match the organization name directory structure. OWNERS_FILE_VALUE={owners_value_chart_name}, DIRECTORY_VALUE:{chart}" + ) + return 35 + + new_entry = f"{category}/{organization}/{chart}" + if packages.get(chart) is not None: + logError( + f"Duplicate chart name detected. Unable to build unique package list. trying to add: {new_entry}, current_value: {packages[chart]}" + ) + return 40 + + packages[chart] = new_entry + + if len(packages.keys()) == 0: + logError("the package map contained no items!") + return 50 + + now = datetime.now(timezone.utc).astimezone().isoformat() + + print( + to_json( + { + "generated": now, + "packages": packages, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/src/pullrequest/metadata.py b/scripts/src/pullrequest/metadata.py new file mode 100644 index 0000000000..436550273f --- /dev/null +++ b/scripts/src/pullrequest/metadata.py @@ -0,0 +1,142 @@ +import json +import re +import sys +from argparse import ArgumentParser + +from reporegex import matchers +from tools import gitutils + +from pullrequest import prartifact + + +class NoMatchesError(Exception): + def __init__(self): + super().__init__( + "no file modifications within the chart submission path were detected in this pr" + ) + + +class MultipleMatchesError(Exception): + def __init__(self, first_match, subsequent_match): + super().__init__( + "pull request contained modifications to files associated with multiple charts" + ) + self.first_match = first_match + self.second_match = subsequent_match + + +def extract_from_path_for_pr(pr_api_url): + """Extracts chart submission metadata from modified files in a pull request. + + This will check all filenames contained in a given PR and try to extract the + category, organization, and chart name from the path. + + Notably, this functionality does NOT pull version information for any submission. + + Args: + pr_api_url (str): URL of the GitHub PR + + Raises: + NoMatchesError: In cases where the PR did not contain any files that + matched the expected format for a chart contribution. + MultipleMatchesError: In cases where the PR contained changes to multiple + charts, which is explicitly forbidden. + + Returns: + Returns the category, organization, and name. E.g. "partner", + "hashicorp", "vault". + """ + + modified_files = prartifact.get_modified_files(pr_api_url) + first_match = None + modification_matcher = re.compile( + matchers.submission_path_matcher( + strict_categories=True, include_version_matcher=False + ) + ) + for file in modified_files: + match = modification_matcher.match(file) + if not match: + continue + + # Store the value of our groupings the first time we match + # so that we can track if it changes. + if not first_match: + first_match = match + continue + + if match.groups() != first_match.groups(): + raise MultipleMatchesError( + first_match=first_match.groups(), + subsequent_match=match.groups(), + ) + + # We didn't find a match, so we need to bail. + if not first_match: + raise NoMatchesError + + cat, org, name = first_match.groups() + # normalize the partners directory to partner + cat = "partner" if cat == "partners" else cat + return cat, org, name + + +def main(): + """CLI for determining the the cat/org/chart being modified. + + Return codes: + 0: No problems. + 10: Multiple matches were found. + 20: No matches were found. + """ + parser = ArgumentParser( + description="""Read the modified files from a GitHub Pull Requests +and determine the category, organization, and chartname being modified. + +This information is extracted from the relative path of the files being +modified. Specifically, this reviews the charts/ directory and looks for the +subsequent path to determine the category, organization, and chart name. +""" + ) + parser.add_argument( + "api_url", + help="The API URL for a given pull request. E.g. https://api.github.com/repos/openshift-helm-charts/charts/pulls/1", + type=str, + ) + parser.add_argument( + "-g", + "--emit-to-github-output", + help="Sends keys and values to the $GITHUB_OUTPUT.", + default=False, + dest="emit_to_github", + action="store_true", + ) + args = parser.parse_args() + try: + category, organization, chartname = extract_from_path_for_pr(args.api_url) + except MultipleMatchesError as e: + print( + f"[Error] {e}", + f"First Match: {e.first_match}", + f"Second Match: {e.second_match}", + file=sys.stderr, + ) + return 10 + except NoMatchesError as e: + print(f"[Error] {e}", file=sys.stderr) + return 20 + + print( + json.dumps( + {"category": category, "organization": organization, "chartname": chartname} + ) + ) + + if args.emit_to_github: + gitutils.add_output("category", category) + gitutils.add_output("organization", organization) + gitutils.add_output("chart-name", chartname) + + +if __name__ == "__main__": + main() diff --git a/scripts/src/pullrequest/prartifact.py b/scripts/src/pullrequest/prartifact.py index a0f74f7ba1..f70e15760a 100644 --- a/scripts/src/pullrequest/prartifact.py +++ b/scripts/src/pullrequest/prartifact.py @@ -1,8 +1,8 @@ -import os -import sys import argparse -import shutil +import os import pathlib +import shutil +import sys import requests diff --git a/scripts/src/pullrequest/prepare_pr_comment.py b/scripts/src/pullrequest/prepare_pr_comment.py index e6faf08678..5a89943fbc 100644 --- a/scripts/src/pullrequest/prepare_pr_comment.py +++ b/scripts/src/pullrequest/prepare_pr_comment.py @@ -1,5 +1,6 @@ import os import sys + from tools import gitutils diff --git a/scripts/src/release/release_info.py b/scripts/src/release/release_info.py index 073f01f3d4..ab9ba21830 100644 --- a/scripts/src/release/release_info.py +++ b/scripts/src/release/release_info.py @@ -4,10 +4,10 @@ Provides get functions for all data in the release_info.json file. """ + import json import os - RELEASE_INFO_FILE = "release/release_info.json" RELEASE_INFOS = {} diff --git a/scripts/src/release/releasechecker.py b/scripts/src/release/releasechecker.py index 135c9b78aa..c8647583cc 100644 --- a/scripts/src/release/releasechecker.py +++ b/scripts/src/release/releasechecker.py @@ -22,21 +22,21 @@ PR_release_image : The name of the image from the version file from main branch. """ - -import re -import os import argparse import json -import semver +import os +import re import sys -from release import release_info -from release import releaser + +import semver from reporegex import matchers +from release import release_info, releaser + sys.path.append("../") from owners import checkuser -from tools import gitutils from pullrequest import prartifact +from tools import gitutils VERSION_FILE = "release/release_info.json" CHARTS_PR_BASE_REPO = gitutils.CHARTS_REPO diff --git a/scripts/src/release/releaser.py b/scripts/src/release/releaser.py index 0f5926b21a..8d0fcc81f1 100644 --- a/scripts/src/release/releaser.py +++ b/scripts/src/release/releaser.py @@ -19,10 +19,12 @@ """ -import os + import argparse -import sys +import os import shutil +import sys + from release import release_info sys.path.append("../") diff --git a/scripts/src/report/get_verify_params.py b/scripts/src/report/get_verify_params.py index 58152d8f2b..a105dbae33 100644 --- a/scripts/src/report/get_verify_params.py +++ b/scripts/src/report/get_verify_params.py @@ -1,6 +1,6 @@ +import argparse import os import sys -import argparse sys.path.append("../") from chartprreview import chartprreview diff --git a/scripts/src/report/report_info.py b/scripts/src/report/report_info.py index 304446cf0e..56fd280dd3 100644 --- a/scripts/src/report/report_info.py +++ b/scripts/src/report/report_info.py @@ -1,8 +1,9 @@ +import json import os +import subprocess import sys + import docker -import json -import subprocess REPORT_ANNOTATIONS = "annotations" REPORT_RESULTS = "results" diff --git a/scripts/src/report/verifier_report.py b/scripts/src/report/verifier_report.py index e168f2a3c5..875444e6a9 100644 --- a/scripts/src/report/verifier_report.py +++ b/scripts/src/report/verifier_report.py @@ -1,17 +1,17 @@ """ -Used by github actions,specifically as part of the charts auto release process defined in +Used by github actions, specifically as part of the charts auto release process defined in .github/workflow/release.yml. Used to loosely determine if a submitted report is valid and has not been tampered with. An invalid valid report: - does not load as a yaml file. -- does not include a "kind" attribute set the "verify-report" . +- does not include a "kind" attribute set to "verify-report". - does not include sections: "tool.metadata", "tool.chart", "results". A tampered report is only determined if the chart-testing check has passed: - certifiedOpenShiftVersions or testOpenShiftVersion contain valid semantic versions. -- certifiedOpenShiftVersions or testOpenShiftVersion specify an OCP version with helm support (>=4.1.0) +- certifiedOpenShiftVersions or testOpenShiftVersion specify an OCP version with helm support (>=4.1.0) - if the has-kubeversion check has also passed - v1.0 profile: - if a valid kubeVersion is specified in the chart it must include the certifiedOpenShiftVersions @@ -24,14 +24,14 @@ """ import sys -import semantic_version +import semantic_version import yaml try: - from yaml import CLoader as Loader + from yaml import CSafeLoader as SafeLoader except ImportError: - from yaml import Loader + from yaml import SafeLoader sys.path.append("../") from report import report_info @@ -55,7 +55,7 @@ def get_report_data(report_path): """ try: with open(report_path) as report_data: - report_content = yaml.load(report_data, Loader=Loader) + report_content = yaml.load(report_data, Loader=SafeLoader) return True, report_content except Exception as err: print(f"Exception 2 loading file: {err}") diff --git a/scripts/src/saforcertadmin/push_secrets.py b/scripts/src/saforcertadmin/push_secrets.py index 9d1ecf4cc4..782a29ac12 100644 --- a/scripts/src/saforcertadmin/push_secrets.py +++ b/scripts/src/saforcertadmin/push_secrets.py @@ -19,14 +19,16 @@ python push_secrets.py -r openshift-helm-charts/sandbox -s CLUSTER_TOKEN -v """ -from base64 import b64encode -from nacl import encoding, public + +import argparse +import json import logging import os import sys -import json +from base64 import b64encode + import requests -import argparse +from nacl import encoding, public sys.path.append("../") from pullrequest import prartifact diff --git a/scripts/src/saforcharttesting/saforcharttesting.py b/scripts/src/saforcharttesting/saforcharttesting.py index ba2ef7b2d7..aab88d0f35 100644 --- a/scripts/src/saforcharttesting/saforcharttesting.py +++ b/scripts/src/saforcharttesting/saforcharttesting.py @@ -1,12 +1,12 @@ -import sys -import time -import os +import argparse import base64 import json -import argparse +import os +import re import subprocess +import sys import tempfile -import re +import time from string import Template namespace_template = """\ diff --git a/scripts/src/signedchart/signedchart.py b/scripts/src/signedchart/signedchart.py index df84c2c8ee..fc852f880a 100644 --- a/scripts/src/signedchart/signedchart.py +++ b/scripts/src/signedchart/signedchart.py @@ -1,15 +1,15 @@ -import sys -import subprocess import base64 import filecmp import os import re +import subprocess +import sys sys.path.append("../") -from report import verifier_report from owners import owners_file from pullrequest import prartifact from reporegex import matchers +from report import verifier_report def check_and_prepare_signed_chart(api_url, report_path, owner_path, key_file_path): diff --git a/scripts/src/tools/gitutils.py b/scripts/src/tools/gitutils.py index 52debd770a..a451ee0c53 100644 --- a/scripts/src/tools/gitutils.py +++ b/scripts/src/tools/gitutils.py @@ -11,10 +11,10 @@ - commit_development_change - directly commits changes to the main branch of the devlopment repository """ - +import json import os import sys -import json + import requests from git import Repo diff --git a/scripts/src/updateindex/updateindex.py b/scripts/src/updateindex/updateindex.py index 62c688f0b1..e0f89ff2be 100644 --- a/scripts/src/updateindex/updateindex.py +++ b/scripts/src/updateindex/updateindex.py @@ -6,17 +6,18 @@ import hashlib import json import os -import requests import sys -import yaml - from datetime import datetime, timezone + +import requests +import yaml from environs import Env try: - from yaml import CLoader as Loader, CDumper as Dumper + from yaml import CDumper as Dumper + from yaml import CLoader as Loader except ImportError: - from yaml import Loader, Dumper + from yaml import Dumper, Loader def _decode_chart_entry(chart_entry_encoded): diff --git a/scripts/src/workflowtesting/checkprforci.py b/scripts/src/workflowtesting/checkprforci.py index 7e862045d4..3dd99a3f0d 100644 --- a/scripts/src/workflowtesting/checkprforci.py +++ b/scripts/src/workflowtesting/checkprforci.py @@ -1,9 +1,10 @@ -import re import argparse import os -import yaml +import re import sys +import yaml + try: from yaml import CLoader as Loader except ImportError: @@ -24,7 +25,6 @@ def check_if_ci_only_is_modified(api_url): re.compile(r"tests/.*"), ] test_files = [ - re.compile(r"tests/functional/step_defs/.*_test_.*"), re.compile(r"tests/functional/behave_features/.*.feature"), ] skip_build_files = [ diff --git a/tests/functional/behave_features/common/utils/chart_certification.py b/tests/functional/behave_features/common/utils/chart_certification.py index d300ea3740..6094c35a03 100644 --- a/tests/functional/behave_features/common/utils/chart_certification.py +++ b/tests/functional/behave_features/common/utils/chart_certification.py @@ -329,7 +329,7 @@ def check_workflow_conclusion( ): try: # Check workflow conclusion - run_id = get_run_id(self.secrets, pr_number) + run_id = get_run_id(self.secrets, WORKFLOW_CERTIFICATION_CI, pr_number) conclusion = get_run_result(self.secrets, run_id) if conclusion == expect_result: logging.info( @@ -735,7 +735,7 @@ def update_chart_version_in_chart_yaml(self, new_version): with open(path, "w") as fd: fd.write(yaml.dump(chart)) except Exception as e: - raise AssertionError("Failed to update version in yaml file") + raise AssertionError(f"Failed to update version in yaml file: {e}") def remove_readme_file(self): with SetDirectory(Path(self.temp_dir.name)): diff --git a/tests/functional/behave_features/common/utils/e2e_templates.py b/tests/functional/behave_features/common/utils/e2e_templates.py new file mode 100644 index 0000000000..6fadbe1e8a --- /dev/null +++ b/tests/functional/behave_features/common/utils/e2e_templates.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +"""String templates to be used for generating various files throughout E2E testing.""" + +# The base OWNERS file string template. +base_owners_file: str = """\ +chart: + name: ${chart_name} + shortDescription: Test chart for testing chart submission workflows. +publicPgpKey: ${public_key} +providerDelivery: ${provider_delivery} +users: +- githubUsername: ${bot_name} +vendor: + label: ${vendor} + name: ${vendor_pretty} +""" diff --git a/tests/functional/behave_features/common/utils/github.py b/tests/functional/behave_features/common/utils/github.py index 52dcbbe19e..0a099a822c 100644 --- a/tests/functional/behave_features/common/utils/github.py +++ b/tests/functional/behave_features/common/utils/github.py @@ -10,16 +10,25 @@ @retry(stop_max_delay=30_000, wait_fixed=1000) -def get_run_id(secrets, pr_number=None): +def get_run_id(secrets, workflow_name: str, pr_number: str = None): + """Queries the GitHub API for the ID of the workflow run associated with a PR. + + Args: + secrets: Used to extract the GitHub authentication credentials. + workflow_name: The value of the 'name' key in a given GitHub Action. + pr_number: The pull request for which to search workflow runs. + + Raises: + Exception in cases where a workflow with that name did not run. + """ + logging.debug(f'getting run id for workflow named "{workflow_name}') pr = get_pr(secrets, pr_number) r = github_api("get", f"repos/{secrets.test_repo}/actions/runs", secrets.bot_token) runs = json.loads(r.text) for run in runs["workflow_runs"]: - if ( - run["head_sha"] == pr["head"]["sha"] - and run["name"] == CERTIFICATION_CI_NAME - ): + if run["head_sha"] == pr["head"]["sha"] and run["name"] == workflow_name: + logging.debug(f'workflow found with id "{run["id"]}"') return run["id"] else: raise Exception("Workflow for the submitted PR did not run.") diff --git a/tests/functional/behave_features/common/utils/owner_file_submission.py b/tests/functional/behave_features/common/utils/owner_file_submission.py new file mode 100644 index 0000000000..f91f44339b --- /dev/null +++ b/tests/functional/behave_features/common/utils/owner_file_submission.py @@ -0,0 +1,379 @@ +# -*- coding: utf-8 -*- +"""Utility class for setting up and manipulating owner file submissions""" + +import os +import logging +import pathlib +import uuid +import git +import json + +from dataclasses import dataclass, field +from tempfile import TemporaryDirectory +from string import Template +from pathlib import Path + +from common.utils.workflow_repo_manager import WorkflowRepoManager +from common.utils.secret import E2ETestSecretOneShot +from common.utils.set_directory import SetDirectory +from common.utils.e2e_templates import base_owners_file +import common.utils.github as github +import common.utils.env as env +import common.utils.setttings as settings + + +@dataclass +class OwnersFileSubmissionsE2ETest: + # Generated at initialization. Identifies each instance uniquely. + # Used for PRs titles, branch names, etc. + uuid: str = field(default_factory=lambda: uuid.uuid4().hex, init=False) + head_sha: str = "" + # Meaningful test name for this test, displayed in PR title + test_name: str = "" + secrets: E2ETestSecretOneShot = field(default_factory=lambda: E2ETestSecretOneShot(), init=False) + old_cwd: str = os.getcwd() + # This is the worktree for this test run. To be used for context management where applicable. + temp_dir: TemporaryDirectory = None + # Whether this is being executed in GitHub Actions. + github_actions: str = os.environ.get("GITHUB_ACTIONS") + # The name of the chart as supplied by the user. + chart_base_name = "" + # The name of the chart as supplied by the user appended with a unique identifier. + chart_name = "" + # The vendor name, e.g. 'redhat' + vendor_label = "" + # The vendor category, e.g. 'partners' + vendor_category = "" + # The vendor's pretty name, e.g. 'Red Hat' + vendor_name = "" + # The PR number in string format for this test run. + pull_request: str = "" + + # Manages the local repository, branches, worktress, etc. + # and facilitates pushes to remotes. + repo_manager: WorkflowRepoManager = field(default_factory=lambda: WorkflowRepoManager(), init=False) + + # Combines this instance's unique ID + the repository base branch hash + branch_base_id: str = None + + def __post_init__(self) -> None: + """Generates identifiers for an instance to use.""" + logging.debug("RedHatOwnersFileSubmissionE2ETest --> __post_init__ called!") + # Credentials and paths + bot_name, bot_token = env.get_bot_name_and_token() + test_repo = settings.TEST_REPO + self.repo_manager.set_auth_token(bot_token) + + # Create a new branch locally from detached HEAD + self.head_sha = self.repo_manager.repo.git.rev_parse("--short", "HEAD") + self.branch_base_id = f"{self.head_sha}-{self.uuid}" + + logging.info(f"base branch identifer: {self.unique_branch()}") + # Create the base branch for this workflow run. + self.repo_manager.checkout_branch(self.unique_branch()) + logging.debug( + f"Active branch name: {self.repo_manager.repo.active_branch.name}" + ) + + # Create the branch on the remote if it doesn't exist. + try: + r = github.github_api("get", f"repos/{test_repo}/branches", bot_token) + except Exception as e: + raise AssertionError( + " ".join( + [ + "Failed to Inintialize Test", + "Unable to get branches from the GitHub API.", + "Is GitHub having an outage?", + f"Error: {e}", + ] + ) + ) + + branches = json.loads(r.text) + branch_names = [branch["name"] for branch in branches] + logging.debug(f"Remote test repo branch names : {branch_names}") + if self.unique_branch() not in branch_names: + logging.info( + f"{test_repo}:{self.unique_branch()} does not exists, creating with local branch" + ) + self.repo_manager.push_branch(test_repo, self.unique_branch()) + + # Create base branch and pr branch. + base_branch, pr_branch = self.workflow_branches(self.test_name) + logging.debug(f"base branch is set to: {base_branch}") + logging.debug(f"pr branch is set to: {pr_branch}") + + # Set "secrets" + # + # NOTE(komish): This is a carry-over from the chart certification CI. + # Not all items are secrets, but it's unclear why these are designated + # as such. Prioritizing consistency for now. + self.secrets.test_repo = test_repo + self.secrets.bot_name = bot_name + self.secrets.bot_token = bot_token + self.secrets.base_branch = base_branch + self.secrets.pr_branch = pr_branch + self.secrets.index_file = "index.yaml" + + self.set_git_username_email(bot_name, settings.GITHUB_ACTIONS_BOT_EMAIL) + self.temp_dir = self.repo_manager.add_worktree() + + def unique_branch(self) -> str: + """Returns this instance's unique branch name.""" + return f"e2e-owners-{self.branch_base_id}" + + def workflow_branches(self, test_name: str) -> (str, str): + """Returns the base_branch and pr_branch names for a given test_name.""" + pretty_test_name = self.test_name.strip().lower().replace(" ", "-") + base_branch = ( + f"{self.unique_branch()}-{pretty_test_name}" + if pretty_test_name + else f"{self.unique_branch()}-test" + ) + return base_branch, f"{base_branch}-pr-branch" + + def cleanup(self): + """Cleans up all artifacts that are created locally and remotely.""" + logging.debug("RedHatOwnersFileSubmissionE2ETest --> cleanup called!") + if self.repo_manager: + try: + self.repo_manager.cleanup() + except Exception as e: + logging.error( + f"failed to execute cleanup of the repo_manager with error: {e}" + ) + + # Teardown step to cleanup branches + if self.temp_dir is not None: + self.temp_dir.cleanup() + + if self.repo_manager: + self.repo_manager.repo.git.worktree("prune") + + def submission_path(self) -> str: # charts/partners/mycompany/mychartname + """Composes the submission path based on the vendor_type, vendor, and chart_name values.""" + return os.path.join( + "charts", self.vendor_category, self.vendor_label, self.chart_name + ) + + def set_git_username_email(self, username: str, email: str): + """Writes the username and email to the repo's .git/config. + + Note that calling this will overwrite repository-local values, which + will supercede global values. + + Args: + username: git username to set + email: git email to set + """ + self.repo_manager.repo.config_writer().set_value( + "user", "name", username + ).release() + self.repo_manager.repo.config_writer().set_value( + "user", "email", email + ).release() + + def create_and_commit_owners_file(self): + self.repo_manager.checkout_branch(self.secrets.base_branch) + self.repo_manager.push_branch(self.secrets.test_repo, self.secrets.base_branch) + self.repo_manager.checkout_branch(self.secrets.pr_branch) + return self._create_and_commit_owners_file( + self.submission_path(), + self.secrets.pr_branch, + self.chart_name, + self.vendor_label, + self.vendor_name, + ) + + def _create_and_commit_owners_file( + self, + chart_directory: str, + pr_branch: str, + chart_name: str, + vendor_label: str, + vendor_name: str, + ): + """Creates an OWNERS file in the PR branch.""" + with SetDirectory(Path(self.temp_dir.name)): + # Create the OWNERS file from the string template + values = { + "bot_name": self.secrets.bot_name, + "vendor": vendor_label, + "vendor_pretty": vendor_name, + "chart_name": chart_name, + "public_key": "null", + "provider_delivery": "false", + } + content = Template(base_owners_file).substitute(values) + logging.debug(f"OWNERS File Content:\n{content}") + pathlib.Path(chart_directory).mkdir(parents=True, exist_ok=True) + with open(f"{chart_directory}/OWNERS", "w+") as fd: + fd.write(content) + + logging.info(f"Push OWNERS file to '{self.secrets.test_repo}:{pr_branch}'") + worktree_repo = git.Repo() + worktree_repo.git.add(f"{chart_directory}/OWNERS") + worktree_repo.git.commit("-m", f"Add {chart_name} OWNERS file") + self.repo_manager.push_branch( + self.secrets.test_repo, pr_branch, repo=worktree_repo + ) + + def send_pull_request(self): + """Sends a pull request to be evaluated for CI.""" + self.pull_request = self._send_pull_request( + self.secrets.test_repo, + self.secrets.base_branch, + self.secrets.pr_branch, + self.secrets.bot_token, + os.environ.get("PR_BODY"), + ) + return self.pull_request + + def _send_pull_request( + self, + remote_repo: str, + base_branch: str, + pr_branch: str, + gh_token: str, + pr_body: str, + ): + """Sends a pull request using the GitHub API. + + The branches should already exist in the remotes. This only sends the + pull request API call referring to two pre-existing branches in + remote_repo. + + Pull requests created with this method are not tracked for cleanup. + When related branches are deleted, GitHub will automatically close the PR. + + Args: + remote_repo: the org/repo string where the PR should be pushed. + base_branch: the branch receiving the PR. + pr_branch: the modified content to send to base_branch. + pr_body: the text to send in the PR body. + + Raises: + AssertionError in cases where the pull requests could not be sent. + + Returns: + The PR number for the generated PR. + """ + data = { + "head": pr_branch, + "base": base_branch, + "title": base_branch, + "body": pr_body, + } + logging.debug(f"Pull Request Body Text: {pr_body}") + logging.info( + f"Create PR from '{remote_repo}:{pr_branch}' to '{remote_repo}:{base_branch}'" + ) + try: + r = github.github_api( + "post", f"repos/{remote_repo}/pulls", gh_token, json=data + ) + except Exception as e: + raise AssertionError( + " ".join( + [ + "Failed to send pull request.", + "Is GitHub having an outage?", + f"Error: {e}", + ] + ) + ) + j = json.loads(r.text) + if "number" not in j: + raise AssertionError(f"error sending pull request, response was: {r.text}") + return j["number"] + + def check_workflow_conclusion( + self, + expected_result: str, + ): + """Checks the input workflow and reports back if expected_result doesn't match reality.""" + + # TODO(komish): When this testing is extended to partner/community files, + # setting workflow_name should be more dynamic. For now, this always looks + # for the Red Hat workflow because those are the only checks that exist. + workflow_name = settings.WORKFLOW_REDHAT_OWNERS_CHECK + + return self._check_workflow_conclusion( + self.pull_request, + workflow_name, + expected_result, + ) + + def _check_workflow_conclusion( + self, + pr_number: str, + workflow_name: str, + expected_result: str, + failure_type="error", + ): + """Checks the conclusion of workflow_name for expected_result via the GitHub API. + + Args: + pr_number: The pull request for which a worfklow should have executed, e.g. '1'. + workflow_name: The name of the workflow whose outcome is relevant + expected_result: The expected conclusion of the workflow. E.g. 'success' + failure_type: Determines how this function treats conclusion mismatches. + Raises if set to 'error'. + + Raises: + AssertionError if the conclusion does not match the expected_result. + + Returns: + run_id: The ID of the workflow + conclusion: The actual conclusion of the workflow. + """ + try: + run_id = github.get_run_id(self.secrets, workflow_name, pr_number) + conclusion = github.get_run_result(self.secrets, run_id) + except Exception as e: + if failure_type == "error": + raise AssertionError(e) + else: + logging.warning(e) + return None, None + + if conclusion == expected_result: + logging.info( + f"PR{pr_number} Workflow run was '{expected_result}' which is expected" + ) + else: + if failure_type == "error": + raise AssertionError( + f"PR{pr_number if pr_number else self.secrets.pr_number} Workflow run was '{conclusion}' which is unexpected, run id: {run_id}" + ) + else: + logging.warning( + f"PR{pr_number if pr_number else self.secrets.pr_number} Workflow run was '{conclusion}' which is unexpected, run id: {run_id}" + ) + + return run_id, conclusion + + def set_chart_name(self, basename): + """Generates a unique chart name from basename and stores both in self. + + Generated values are appended to the basename. + + Returns: + The generated chart name value. + """ + self.chart_base_name = basename + self.chart_name = self.append_unique_id(basename) + logging.debug( + f'Caller provided chart name "{self.chart_base_name}", generated unique identifier: "{self.chart_name}"' + ) + return self.chart_name + + def append_unique_id(self, base) -> str: + """Appends this class' unique identifier, and optionally, the pr number.""" + # unique string based on uuid.uuid4() + suffix = self.uuid + if "PR_NUMBER" in os.environ: + pr_num = os.environ["PR_NUMBER"] + suffix = f"{suffix}-{pr_num}" + return f"{base}-{suffix}" diff --git a/tests/functional/behave_features/common/utils/setttings.py b/tests/functional/behave_features/common/utils/setttings.py index 45caafc860..b3e9c64e15 100644 --- a/tests/functional/behave_features/common/utils/setttings.py +++ b/tests/functional/behave_features/common/utils/setttings.py @@ -8,7 +8,11 @@ PROD_REPO = "openshift-helm-charts/charts" # The prod branch where we store all chart files PROD_BRANCH = "main" -# This is used to find chart certification workflow run id +# (Deprecated) This is used to find chart certification workflow run id CERTIFICATION_CI_NAME = "CI" +# (Replaces the above) The name of the workflow for certification, used to get its ID. +WORKFLOW_CERTIFICATION_CI = "CI" +# The name of the workflow for Red Hat OWNERS check submissions, used to get its ID. +WORKFLOW_REDHAT_OWNERS_CHECK = "Red Hat OWNERS Files" # GitHub actions bot email for git email GITHUB_ACTIONS_BOT_EMAIL = "41898282+github-actions[bot]@users.noreply.github.com" diff --git a/tests/functional/behave_features/common/utils/workflow_repo_manager.py b/tests/functional/behave_features/common/utils/workflow_repo_manager.py new file mode 100644 index 0000000000..98f6ebd685 --- /dev/null +++ b/tests/functional/behave_features/common/utils/workflow_repo_manager.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +"""Utility class for managing local/remote git branches and worktrees""" + +import os +import logging +import git + +from tempfile import TemporaryDirectory + +import common.utils.github as github + + +class RepoManagementError(Exception): + pass + + +class WorkflowRepoManager: + # Keep a log of things created so we can clean them up. + __local_branches_created: [str] = [] + __local_worktrees_created: [TemporaryDirectory] = [] + # (remote, branch), e.g. ('openshift-helm-charts/charts, 'my-pr-branch') + __remote_branches_created: [(str, str)] = [] + + # The token to use for GitHub API operations. + __authtoken: str = "" + + # The branch at repo initialization. On Cleanup, we return to this branch + # before we remove locally generated branches. + original_branch: str = None + + # Working directory at instantiation. + old_cwd: str = os.getcwd() + + # The repository at working directory. + repo: git.Repo = None + + def __init__(self): + logging.debug(f"{self} --> __init__ called!") + + try: + self.repo = git.Repo() + except git.InvalidGitRepositoryError as e: + raise RepoManagementError( + "Unable to initialize git repository. Is the current directory a git repo?" + ) from e + + self.original_branch = self.repo.active_branch.name + + def set_auth_token(self, token: str): + """Sets the github API token.""" + self.__authtoken = token + + def push_branch(self, remote_name: str, branch_name: str, repo: git.Repo = None): + """Pushes the input branch_name to remote_name. + + The branch must exist locally before this is called, as it is not + created by this method. + + Args: + remote_name: the name of the remote in organization/repository format. + branch_name: the branch to push. + repo: if set, overrides the use of the internal repo. + + Raises: + RepoManagementError: When failing to push a branch. + """ + r = repo if repo is not None else self.repo + try: + r.git.push( + f"https://x-access-token:{self.__authtoken}@github.com/{remote_name}", + f"HEAD:refs/heads/{branch_name}", + "-f", + ) + except (git.GitCommandError, ValueError) as e: + raise RepoManagementError("Unable to push branch to remote") from e + self.__remote_branches_created.append((remote_name, branch_name)) + + def checkout_branch(self, branch_name: str): + """Checks out a branch at branch_name and switches to it immediately. + + Branches created with this method are stored internally + and cleaned up when cleanup() is called. + + Args: + branch_name: The branch name to create. + + Raises: + RepoManagementError: When creating a branch locally. + """ + try: + self.repo.git.checkout("-b", branch_name) + except (git.GitCommandError, ValueError) as e: + raise RepoManagementError("Unable to create branch") from e + self.__local_branches_created.append(branch_name) + + def add_worktree(self) -> TemporaryDirectory: + """Creates a worktree at a generated tempdir. + + Worktrees created with this method are stored internally + and cleaned up when cleanup() is called. + + Returns: + The TemporaryDirectory for the worktree that was requested. + + Raise: + RepoManagementError: When creating a worktree locally. + """ + worktree_dir = TemporaryDirectory(prefix="worktree-") + + try: + self.repo.git.worktree("add", "--detach", worktree_dir.name, "HEAD") + except (git.GitCommandError, ValueError) as e: + raise RepoManagementError("Unable to create worktree") from e + self.__local_worktrees_created.append(worktree_dir) + + return worktree_dir + + def cleanup_local_branches(self): + """Cleans up branches created by this repo manager.""" + for br in self.__local_branches_created: + try: + logging.info(f'Cleaning up generated local branch: "{br}"') + self.repo.git.branch("-D", br) + except (git.GitCommandError, ValueError) as e: + logging.warn( + f'local branch "{br}" could not be deleted, potentially because it did not exist. Error: {e}' + ) + self.__local_branches_created = [] + + def cleanup_worktrees(self): + """Cleans up local worktrees created by this repo manager.""" + for wt in self.__local_worktrees_created: + logging.info(f'Cleaning up generated local worktree: "{wt}"') + try: + self.repo.git.worktree("remove", wt.name) + except (git.GitCommandError, ValueError) as e: + logging.warn( + f'local worktree "{wt}" could not be deleted, potentially because it did not exist. Error: {e}' + ) + self.__local_worktrees_created = [] + + def cleanup_remote_branches(self): + """Cleans up remote branches created by this repo manager.""" + for rbr in self.__remote_branches_created: + remote = rbr[0] + branch = rbr[1] + logging.info(f'Cleaning up branch "{branch}" from remote "{remote}') + try: + self.repo.git.push( + f"https://x-access-token:{self.__authtoken}@github.com/{remote}", + "--delete", + f"refs/heads/{branch}", + ) + except (git.GitCommandError, ValueError) as e: + logging.warn( + f'remote branch "{branch}" could not be deleted from {remote}, potentially because it did not exist. Error: {e}' + ) + self.__remote_branches_created = [] + + def cleanup(self): + """Cleans up resources created using this instance. + + Locally created branches and worktrees are removed. Exceptions in removing + these will emit a log line at the warn log level. + """ + logging.debug(f"{self} --> cleanup called!") + self.repo.git.checkout(self.original_branch) + self.cleanup_local_branches() + self.cleanup_worktrees() + self.cleanup_remote_branches() diff --git a/tests/functional/features/HC-01_chart_src_without_report.feature b/tests/functional/features/HC-01_chart_src_without_report.feature deleted file mode 100644 index a02621e71c..0000000000 --- a/tests/functional/features/HC-01_chart_src_without_report.feature +++ /dev/null @@ -1,31 +0,0 @@ -Feature: Chart source submission without report - Partners, redhat and community users can publish their chart by submitting - error-free chart in source format without a report. - - Examples: - | chart_path | - | tests/data/vault-0.17.0.tgz | - - Scenario Outline: [HC-01-001] A partner or redhat associate submits an error-free chart source - Given the vendor has a valid identity as - And an error-free chart source is used in - When the user sends a pull request with the chart - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - | redhat | redhat | - - Scenario Outline: [HC-01-002] A community user submits an error-free chart source without report - Given the vendor has a valid identity as - And an error-free chart source is used in - When the user sends a pull request with the chart - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/features/HC-02_chart_tar_without_report.feature b/tests/functional/features/HC-02_chart_tar_without_report.feature deleted file mode 100644 index b49da7b030..0000000000 --- a/tests/functional/features/HC-02_chart_tar_without_report.feature +++ /dev/null @@ -1,31 +0,0 @@ -Feature: Chart tarball submission without report - Partners, redhat and community users can publish their chart by submitting - error-free chart in tarball format without a report. - - Examples: - | chart_path | - | tests/data/vault-0.17.0.tgz | - - Scenario Outline: [HC-02-001] A partner or redhat associate submits an error-free chart tarball - Given the vendor has a valid identity as - And an error-free chart tarball is used in - When the user sends a pull request with the chart - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - | redhat | redhat | - - Scenario Outline: [HC-02-002] A community user submits an error-free chart tarball without report - Given the vendor has a valid identity as - And an error-free chart tarball is used in - When the user sends a pull request with the chart - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/features/HC-03_chart_verifier_comes_back_with_failures.feature b/tests/functional/features/HC-03_chart_verifier_comes_back_with_failures.feature deleted file mode 100644 index 5f21ab4ae3..0000000000 --- a/tests/functional/features/HC-03_chart_verifier_comes_back_with_failures.feature +++ /dev/null @@ -1,33 +0,0 @@ -Feature: Chart verifier comes back with a failure - Partners, redhat or community user submit charts which does not contain README file - - Examples: - | chart_path | - | tests/data/vault-0.17.0.tgz | - - Scenario Outline: [HC-03-001] A partner or community user submits a chart which does not contain a readme file - Given the vendor has a valid identity as - And chart source is used in - And README file is missing in the chart - When the user pushed the chart and created pull request - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | partners | hashicorp | Chart does not have a README | - | community | redhat | Community charts require maintainer review and approval | - - Scenario Outline: [HC-03-002] A redhat user submits a chart which does not contain a readme file - Given the vendor has a valid identity as - And chart source is used in - And README file is missing in the chart - When the user pushed the chart and created pull request - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart with correct providerType - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | redhat | redhat | - diff --git a/tests/functional/features/HC-04_invalid_url_in_the_report.feature b/tests/functional/features/HC-04_invalid_url_in_the_report.feature deleted file mode 100644 index cde6a88330..0000000000 --- a/tests/functional/features/HC-04_invalid_url_in_the_report.feature +++ /dev/null @@ -1,26 +0,0 @@ -Feature: Report contains an invalid URL - Partners, redhat and community users submits only report with an invalid URL - - Examples: - | report_path | - | tests/data/report.yaml | - - Scenario Outline: [HC-04-001] A user submits a report with an invalid url - Given the vendor has a valid identity as - And a is provided - And the report contains an - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | invalid_url | message | - | partners | hashicorp | example.com/vault-0.13.0.tgz | Missing schema in URL | - | partners | hashicorp | htts://example.com/vault-0.13.0.tgz | Invalid schema | - | partners | hashicorp | https:example.comvault-0.13.0.tgz | Invalid URL | - | redhat | redhat | example.com/vault-0.13.0.tgz | Missing schema in URL | - | redhat | redhat | htts://example.com/vault-0.13.0.tgz | Invalid schema | - | redhat | redhat | https:example.comvault-0.13.0.tgz | Invalid URL | - | community | redhat | example.com/vault-0.13.0.tgz | Missing schema in URL | - | community | redhat | htts://example.com/vault-0.13.0.tgz | Invalid schema | - | community | redhat | https:example.comvault-0.13.0.tgz | Invalid URL | diff --git a/tests/functional/features/HC-05_pr_includes_a_file_which_is_not_chart_related.feature b/tests/functional/features/HC-05_pr_includes_a_file_which_is_not_chart_related.feature deleted file mode 100644 index 231af26180..0000000000 --- a/tests/functional/features/HC-05_pr_includes_a_file_which_is_not_chart_related.feature +++ /dev/null @@ -1,20 +0,0 @@ -Feature: PR includes a non chart related file - Partners, redhat or community user submit charts which includes a file which is not part of the chart - - Examples: - | chart_path | message | - | tests/data/vault-0.17.0.tgz | PR includes one or more files not related to charts | - - Scenario Outline: [HC-05-001] A user submits a chart with non chart related file - Given the vendor has a valid identity as - And chart source is used in - And user adds a non chart related file - When the user sends a pull request with both chart and non related file - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - | redhat | redhat | - | community | redhat | diff --git a/tests/functional/features/HC-06_provider_delivery_control.feature b/tests/functional/features/HC-06_provider_delivery_control.feature deleted file mode 100644 index 2f7462035a..0000000000 --- a/tests/functional/features/HC-06_provider_delivery_control.feature +++ /dev/null @@ -1,52 +0,0 @@ -Feature: Report only submission with provider control settings - Partners can prevent publication of their chart by submitting - error-free report that was generated by chart-verifier and with - prvider controlled delivery set in the report and the OWNERS file. - - Examples: - | report_path | - | tests/data/report.yaml | - - @external-feedback - Scenario Outline: [HC-06-001] A partner associate submits an error-free report with provider controlled delivery - Given the vendor has a valid identity as - And provider delivery control is set to in the OWNERS file - And a is provided - And provider delivery control is set to in the report - When the user sends a pull request with the report - Then the user sees the pull request is merged - And the is updated with an entry for the submitted chart - - Examples: - | vendor_type | vendor | index_file | provider_control_owners | provider_control_report | - | partners | hashicorp | unpublished-certified-charts.yaml | true | true | - - @external-feedback - Scenario Outline: [HC-06-002] A partner associate submits an error-free report and chart with provider controlled delivery - Given the vendor has a valid identity as - And provider delivery control is set to in the OWNERS file - And an error-free chart tarball is used in and report in - And provider delivery control is set to in the report - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | chart_path | provider_control_owners | provider_control_report | message | - | partners | hashicorp | tests/data/vault-0.17.0.tgz | true | true | OWNERS file and/or report indicate provider controlled delivery but pull request is not report only. | - - @external-feedback - Scenario Outline: [HC-06-003] A partner associate submits an error-free report with inconsistent provider controlled delivery setting - Given the vendor has a valid identity as - And provider delivery control is set to in the OWNERS file - And a is provided - And provider delivery control is set to and a package digest is in the report - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | provider_control_owners | provider_control_report | package_digest_set | message | - | partners | hashicorp | true | false | true | OWNERS file indicates provider controlled delivery but report does not. | - | partners | hashicorp | false | true | true | Report indicates provider controlled delivery but OWNERS file does not. | - | partners | hashicorp | true | true | false | Provider delivery control requires a package digest in the report. | diff --git a/tests/functional/features/HC-07_report_and_chart_src.feature b/tests/functional/features/HC-07_report_and_chart_src.feature deleted file mode 100644 index 8cce8b8b85..0000000000 --- a/tests/functional/features/HC-07_report_and_chart_src.feature +++ /dev/null @@ -1,31 +0,0 @@ -Feature: Chart source submission with report - Partners, redhat and community users can publish their chart by submitting - error-free chart in source format with a report. - - Examples: - | chart_path | report_path | - | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | - - Scenario Outline: [HC-07-001] A partner or redhat associate submits an error-free chart source with report - Given the vendor has a valid identity as - And an error-free chart source is used in and report in - When the user sends a pull request with the chart and report - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - | redhat | redhat | - - Scenario Outline: [HC-07-002] A community user submits an error-free chart source with report - Given the vendor has a valid identity as - And an error-free chart source is used in and report in - When the user sends a pull request with the chart and report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/features/HC-08_report_and_chart_tar.feature b/tests/functional/features/HC-08_report_and_chart_tar.feature deleted file mode 100644 index 5899cbd4d9..0000000000 --- a/tests/functional/features/HC-08_report_and_chart_tar.feature +++ /dev/null @@ -1,31 +0,0 @@ -Feature: Chart tarball submission with report - Partners, redhat and community users can publish their chart by submitting - error-free chart in tarball format with a report. - - Examples: - | chart_path | report_path | - | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | - - Scenario Outline: [HC-08-001] A partner or redhat associate submits an error-free chart tarball with report - Given the vendor has a valid identity as - And an error-free chart tarball is used in and report in - When the user sends a pull request with the chart and report - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - | redhat | redhat | - - Scenario Outline: [HC-08-002] A community user submits an error-free chart tarball with report - Given the vendor has a valid identity as - And an error-free chart tarball is used in and report in - When the user sends a pull request with the chart and report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/features/HC-09_report_in_json_format.feature b/tests/functional/features/HC-09_report_in_json_format.feature deleted file mode 100644 index 4cf0c2ba0e..0000000000 --- a/tests/functional/features/HC-09_report_in_json_format.feature +++ /dev/null @@ -1,20 +0,0 @@ -Feature: Report only submission in json format - Partners, redhat and community users trying to publish chart by submitting report in json format - - Examples: - | report_path | message | - | tests/data/report.json | One of these must be modified: report, chart source, or tarball | - - - Scenario Outline: [HC-09-001] An user submits an report in json format - Given the vendor has a valid identity as - And report is used in - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - | redhat | redhat | - | community | redhat | diff --git a/tests/functional/features/HC-10_report_only_edited.feature b/tests/functional/features/HC-10_report_only_edited.feature deleted file mode 100644 index 269fbaf9b3..0000000000 --- a/tests/functional/features/HC-10_report_only_edited.feature +++ /dev/null @@ -1,24 +0,0 @@ -Feature: Edited report only submission - Partners, redhat and community users attempt to publish their chart by submitting - report that was edited after it was generated by chart-verifier. - - Examples: - | vendor_type | vendor | report_path | - | partners | hashicorp | tests/data/report.yaml | - | redhat | redhat | tests/data/report.yaml | - | community | redhat | tests/data/report.yaml | - - Scenario Outline: [HC-10-001] A partner or redhat associate submits an edited report - Given the vendor has a valid identity as - And a is provided - And the report includes and OpenshiftVersion values and chart value - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | tested | supported | kubeversion | message | - | 4.9 | 4.6-4.9 | >=1.20.0 | is not a valid semantic version | - | 4.0 | >=4.7 | >=1.20.0 | is not a supported OpenShift version | - | 4.6 | >=4.7 | >=1.20.0 | not within specified kube-versions | - | 4.8 | >=4.7 | >=1.21.0 | does not match supportedOpenShiftVersions | \ No newline at end of file diff --git a/tests/functional/features/HC-11_report_with_missing_checks.feature b/tests/functional/features/HC-11_report_with_missing_checks.feature deleted file mode 100644 index 6cf31206cb..0000000000 --- a/tests/functional/features/HC-11_report_with_missing_checks.feature +++ /dev/null @@ -1,22 +0,0 @@ -Feature: Report does not include a check - Partners, redhat and community users submits only report which does not include full set of checks - - Examples: - | report_path | - | tests/data/report.yaml | - - Scenario Outline: [HC-11-001] A user submits a report with missing checks - Given the vendor has a valid identity as - And a is provided - And the report has a missing - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | check | message | - | partners | hashicorp | v1.0/helm-lint | Missing mandatory check : v1.0/helm-lint | - # Commented this scenario, since it is failing , raised bug : https://issues.redhat.com/browse/HELM-289 , we can uncomment again when the issue fixed - #| redhat | redhat | v1.0/helm-lint | Missing mandatory check : v1.0/helm-lint | - | community | redhat | v1.0/helm-lint | Missing mandatory check : v1.0/helm-lint | - | partners | hashicorp | v1.0/not-contains-crds | Missing mandatory check : v1.0/not-contains-crds | diff --git a/tests/functional/features/HC-12_report_without_chart.feature b/tests/functional/features/HC-12_report_without_chart.feature deleted file mode 100644 index 9f228eca8e..0000000000 --- a/tests/functional/features/HC-12_report_without_chart.feature +++ /dev/null @@ -1,31 +0,0 @@ -Feature: Report only submission - Partners, redhat and community users can publish their chart by submitting - error-free report that was generated by chart-verifier. - - Examples: - | report_path | - | tests/data/report.yaml | - - - Scenario Outline: [HC-12-001] A partner or redhat associate submits an error-free report - Given the vendor has a valid identity as - And an error-free report is used in - When the user sends a pull request with the report - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - | redhat | redhat | - - Scenario Outline: [HC-12-002] A community user submits an error-free report - Given the vendor has a valid identity as - And an error-free report is used in - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | \ No newline at end of file diff --git a/tests/functional/features/HC-13_sha_value_does_not_match.feature b/tests/functional/features/HC-13_sha_value_does_not_match.feature deleted file mode 100644 index 5ad4acda41..0000000000 --- a/tests/functional/features/HC-13_sha_value_does_not_match.feature +++ /dev/null @@ -1,21 +0,0 @@ -Feature: SHA value in the report does not match - Partners, redhat and community users submits chart tar with report - where chart sha does not match with sha value digests.chart in the report - - Examples: - | chart_path | report_path | - | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | - - Scenario Outline: [HC-13-001] A user submits a chart tarball with report - Given the vendor has a valid identity as - And a chart tarball is used in and report in - And the report contains an - When the user sends a pull request with the chart tar and report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | error | message | - | partners | hashicorp | sha_mismatch | Digest is not matching | - | redhat | redhat | sha_mismatch | Digest is not matching | - | community | redhat | sha_mismatch | Digest is not matching | diff --git a/tests/functional/features/HC-14_user_submits_chart_with_errors.feature b/tests/functional/features/HC-14_user_submits_chart_with_errors.feature deleted file mode 100644 index b88218be64..0000000000 --- a/tests/functional/features/HC-14_user_submits_chart_with_errors.feature +++ /dev/null @@ -1,33 +0,0 @@ -Feature: Chart submission with errors - Partners, redhat or community user submit charts which result in errors - - Examples: - | vendor_type | vendor | chart | version | chart_path | - | partners | hashicorp | vault | 0.17.0 | tests/data/vault-0.17.0.tgz | - | redhat | redhat | vault | 0.17.0 | tests/data/vault-0.17.0.tgz | - | community | redhat | vault | 0.17.0 | tests/data/vault-0.17.0.tgz | - - Scenario Outline: [HC-14-001] An unauthorized user submits a chart - Given A wants to submit a chart in - And of wants to submit of - And the user creates a branch to add a new chart version - When the user sends a pull request with the chart - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | message | user | - | is not allowed to submit the chart on behalf of | unauthorized | - - Scenario Outline: [HC-14-002] An authorized user submits a chart with incorrect version - Given An authorized user wants to submit a chart in - And of wants to submit of - And Chart.yaml specifies a - And the user creates a branch to add a new chart version - When the user sends a pull request with the chart - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | message | bad_version | - | doesn't match the directory structure | 9.9.9 | diff --git a/tests/functional/features/HC-15_check_submitted_charts.feature b/tests/functional/features/HC-15_check_submitted_charts.feature deleted file mode 100644 index 2806bceb3d..0000000000 --- a/tests/functional/features/HC-15_check_submitted_charts.feature +++ /dev/null @@ -1,15 +0,0 @@ -Feature: Check submitted charts - New Openshift or chart-verifier will trigger (automatically or manually) a recursive checking on - existing submitted charts under `charts/` directory with the specified Openshift and chart-verifier - version. - - Besides, during workflow development, engineers would like to check if the changes will break checks - on existing submitted charts. - - Scenario: [HC-15-001] A new Openshift or chart-verifier version is specified either by a cron job or manually - Given there's a github workflow for testing existing charts - When workflow for testing existing charts is triggered - And a new Openshift or chart-verifier version is specified - And the vendor type is specified, e.g. partner, and/or redhat - Then submission tests are run for existing charts - And all results are reported back to the caller diff --git a/tests/functional/features/HC-16_chart_test_takes_more_than_30mins.feature b/tests/functional/features/HC-16_chart_test_takes_more_than_30mins.feature deleted file mode 100644 index f76e5c679b..0000000000 --- a/tests/functional/features/HC-16_chart_test_takes_more_than_30mins.feature +++ /dev/null @@ -1,31 +0,0 @@ -Feature: Chart test takes longer time and exceeds default timeout - Partners, redhat or community user submit charts which result in errors - - Examples: - | chart_path | - | tests/data/vault-test-timeout-0.17.0.tgz | - - Scenario Outline: [HC-16-001] A partner or community user submits chart that takes more than 30 mins - Given the vendor has a valid identity as - And an error-free chart tarball is used in - When the user sends a pull request with the chart - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | partners | hashicorp | (timeout has expired\|timed out waiting for the condition) | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | - - Scenario Outline: [HC-16-002] A redhat associate submits a chart that takes more than 30 mins - Given the vendor has a valid identity as - And an error-free chart tarball is used in - When the user sends a pull request with the chart - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | redhat | redhat | - diff --git a/tests/functional/features/HC-16_dash_in_version.feature b/tests/functional/features/HC-16_dash_in_version.feature deleted file mode 100644 index 9d65c0c8fe..0000000000 --- a/tests/functional/features/HC-16_dash_in_version.feature +++ /dev/null @@ -1,16 +0,0 @@ -Feature: Report only submission - Partners, redhat and community users can publish their chart by submitting - error-free report that was generated by chart-verifier. - - Scenario Outline: [HC-16-001] A partner or redhat associate submits report only with dash in chart version - Given the vendor has a valid identity as - And an error-free report is used in - When the user sends a pull request with the report - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - - Examples: - | vendor_type | vendor | report_path | - | partners | redhat | tests/data/HC-16/dash-in-version/partner/report.yaml | - | redhat | redhat | tests/data/HC-16/dash-in-version/redhat/report.yaml | - diff --git a/tests/functional/features/smoke/README.md b/tests/functional/features/smoke/README.md deleted file mode 100644 index 18e65318bc..0000000000 --- a/tests/functional/features/smoke/README.md +++ /dev/null @@ -1,15 +0,0 @@ -#Smoke Test Suite - -These feature files contains all the smoke tests (a subset of the all the feature files present under "features" directory). - -Smoke tests will run as part of the any workflow change done in the certifcation flow. - -This test suite cover the basic features of chart certification flow. Also tries to touch each verndor_type ie partners, redhat and community. - -Many scenarios contains either partners or redhat path as they mostly share a common code. - -## Guideline for adding new testcases - -Motivation to design this test suite was to minimize testing time and any future test addition to this test suite should not exceed testing time limit of 30 minutes. - -Try to avoid expensive tests which require chart to be installed in certification cluster. \ No newline at end of file diff --git a/tests/functional/features/smoke/chart_src_without_report.feature b/tests/functional/features/smoke/chart_src_without_report.feature deleted file mode 100644 index d2cf9cf0b2..0000000000 --- a/tests/functional/features/smoke/chart_src_without_report.feature +++ /dev/null @@ -1,19 +0,0 @@ -Feature: Chart source submission without report - Partners, redhat and community users can publish their chart by submitting - error-free chart in source format without a report. - - Examples: - | chart_path | - | tests/data/vault-0.17.0.tgz | - - Scenario Outline: A partner or redhat associate submits an error-free chart source - Given the vendor has a valid identity as - And an error-free chart source is used in - When the user sends a pull request with the chart - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | partners | hashicorp | diff --git a/tests/functional/features/smoke/chart_tar_without_report.feature b/tests/functional/features/smoke/chart_tar_without_report.feature deleted file mode 100644 index db5ceee875..0000000000 --- a/tests/functional/features/smoke/chart_tar_without_report.feature +++ /dev/null @@ -1,20 +0,0 @@ -Feature: Chart tarball submission without report - Partners, redhat and community users can publish their chart by submitting - error-free chart in tarball format without a report. - - Examples: - | chart_path | - | tests/data/vault-0.17.0.tgz | - - Scenario Outline: A partner or redhat associate submits an error-free chart tarball - Given the vendor has a valid identity as - And an error-free chart tarball is used in - When the user sends a pull request with the chart - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | redhat | redhat | - diff --git a/tests/functional/features/smoke/chart_verifier_comes_back_with_failures.feature b/tests/functional/features/smoke/chart_verifier_comes_back_with_failures.feature deleted file mode 100644 index b12cb39ce5..0000000000 --- a/tests/functional/features/smoke/chart_verifier_comes_back_with_failures.feature +++ /dev/null @@ -1,19 +0,0 @@ -Feature: Chart verifier comes back with a failure - Partners, redhat or community user submit charts which does not contain README file - - Examples: - | chart_path | - | tests/data/vault-0.17.0.tgz | - - Scenario Outline: A partner or community user submits a chart which does not contain a readme file - Given the vendor has a valid identity as - And chart source is used in - And README file is missing in the chart - When the user pushed the chart and created pull request - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | partners | hashicorp | Chart does not have a README | - diff --git a/tests/functional/features/smoke/invalid_url_in_the_report.feature b/tests/functional/features/smoke/invalid_url_in_the_report.feature deleted file mode 100644 index ed1e577aef..0000000000 --- a/tests/functional/features/smoke/invalid_url_in_the_report.feature +++ /dev/null @@ -1,20 +0,0 @@ -Feature: Report contains an invalid URL - Partners, redhat and community users submits only report with an invalid URL - - Examples: - | report_path | - | tests/data/report.yaml | - - Scenario Outline: A user submits a report with an invalid url - Given the vendor has a valid identity as - And a is provided - And the report contains an - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | invalid_url | message | - | partners | hashicorp | example.com/vault-0.13.0.tgz | Missing schema in URL | - | redhat | redhat | htts://example.com/vault-0.13.0.tgz | Invalid schema | - | community | redhat | https:example.comvault-0.13.0.tgz | Invalid URL | diff --git a/tests/functional/features/smoke/pr_includes_a_file_which_is_not_chart_related.feature b/tests/functional/features/smoke/pr_includes_a_file_which_is_not_chart_related.feature deleted file mode 100644 index 66e469648d..0000000000 --- a/tests/functional/features/smoke/pr_includes_a_file_which_is_not_chart_related.feature +++ /dev/null @@ -1,18 +0,0 @@ -Feature: PR includes a non chart related file - Partners, redhat or community user submit charts which includes a file which is not part of the chart - - Examples: - | chart_path | message | - | tests/data/vault-0.17.0.tgz | PR includes one or more files not related to charts | - - Scenario Outline: A user submits a chart with non chart related file - Given the vendor has a valid identity as - And chart source is used in - And user adds a non chart related file - When the user sends a pull request with both chart and non related file - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | - | partners | hashicorp | diff --git a/tests/functional/features/smoke/provider_delivery_control.feature b/tests/functional/features/smoke/provider_delivery_control.feature deleted file mode 100644 index 916f4b5e82..0000000000 --- a/tests/functional/features/smoke/provider_delivery_control.feature +++ /dev/null @@ -1,22 +0,0 @@ -Feature: Report only submission with provider control settings - Partners can prevent publication of their chart by submitting - error-free report that was generated by chart-verifier and with - provider controlled delivery set in the report and the OWNERS file. - - Examples: - | report_path | - | tests/data/report.yaml | - - - Scenario Outline: A partner associate submits an error-free report with provider controlled delivery - Given the vendor has a valid identity as - And provider delivery control is set to in the OWNERS file - And an error-free report is used in - And provider delivery control is set to in the report - When the user sends a pull request with the report - Then the user sees the pull request is merged - And the is updated with an entry for the submitted chart - - Examples: - | vendor_type | vendor | index_file | provider_control_owners | provider_control_report | - | partners | hashicorp | unpublished-certified-charts.yaml | true | true | diff --git a/tests/functional/features/smoke/report_and_chart_src.feature b/tests/functional/features/smoke/report_and_chart_src.feature deleted file mode 100644 index e052f8b0ac..0000000000 --- a/tests/functional/features/smoke/report_and_chart_src.feature +++ /dev/null @@ -1,30 +0,0 @@ -Feature: Chart source submission with report - Partners, redhat and community users can publish their chart by submitting - error-free chart in source format with a report. - - Examples: - | chart_path | report_path | - | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | - - Scenario Outline: A partner or redhat associate submits an error-free chart source with report - Given the vendor has a valid identity as - And an error-free chart source is used in and report in - When the user sends a pull request with the chart and report - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - - Scenario Outline: A community user submits an error-free chart source with report - Given the vendor has a valid identity as - And an error-free chart source is used in and report in - When the user sends a pull request with the chart and report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/features/smoke/report_and_chart_tar.feature b/tests/functional/features/smoke/report_and_chart_tar.feature deleted file mode 100644 index 094633dd47..0000000000 --- a/tests/functional/features/smoke/report_and_chart_tar.feature +++ /dev/null @@ -1,30 +0,0 @@ -Feature: Chart tarball submission with report - Partners, redhat and community users can publish their chart by submitting - error-free chart in tarball format with a report. - - Examples: - | chart_path | report_path | - | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | - - Scenario Outline: A partner or redhat associate submits an error-free chart tarball with report - Given the vendor has a valid identity as - And an error-free chart tarball is used in and report in - When the user sends a pull request with the chart and report - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - And a release is published with corresponding report and chart tarball - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - - Scenario Outline: A community user submits an error-free chart tarball with report - Given the vendor has a valid identity as - And an error-free chart tarball is used in and report in - When the user sends a pull request with the chart and report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/features/smoke/report_only_edited.feature b/tests/functional/features/smoke/report_only_edited.feature deleted file mode 100644 index b3be9730e6..0000000000 --- a/tests/functional/features/smoke/report_only_edited.feature +++ /dev/null @@ -1,18 +0,0 @@ -Feature: Edited report only submission - Partners, redhat and community users attempt to publish their chart by submitting - report that was edited after it was generated by chart-verifier. - - Scenario Outline: A partner or redhat associate submits an edited report - Given the vendor has a valid identity as - And a is provided - And the report includes and OpenshiftVersion values and chart value - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | report_path | tested | supported | kubeversion | message | - | partners | hashicorp | tests/data/report.yaml | 4.9 | 4.6-4.9 | >=1.20.0 | is not a valid semantic version | - | redhat | redhat | tests/data/report.yaml | 4.0 | >=4.7 | >=1.20.0 | is not a supported OpenShift version | - | community | redhat | tests/data/report.yaml | 4.6 | >=4.7 | >=1.20.0 | not within specified kube-versions | - | partners | hashicorp | tests/data/report.yaml | 4.8 | >=4.7 | >=1.21.0 | does not match supportedOpenShiftVersions | \ No newline at end of file diff --git a/tests/functional/features/smoke/report_with_missing_checks.feature b/tests/functional/features/smoke/report_with_missing_checks.feature deleted file mode 100644 index 433429b8ee..0000000000 --- a/tests/functional/features/smoke/report_with_missing_checks.feature +++ /dev/null @@ -1,18 +0,0 @@ -Feature: Report does not include a check - Partners, redhat and community users submits only report which does not include full set of checks - - Examples: - | report_path | - | tests/data/report.yaml | - - Scenario Outline: A user submits a report with missing checks - Given the vendor has a valid identity as - And a is provided - And the report has a missing - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | check | message | - | partners | hashicorp | v1.0/helm-lint | Missing mandatory check : v1.0/helm-lint | diff --git a/tests/functional/features/smoke/report_without_chart.feature b/tests/functional/features/smoke/report_without_chart.feature deleted file mode 100644 index d6ecd5b3ce..0000000000 --- a/tests/functional/features/smoke/report_without_chart.feature +++ /dev/null @@ -1,30 +0,0 @@ -Feature: Report only submission - Partners, redhat and community users can publish their chart by submitting - error-free report that was generated by chart-verifier. - - Examples: - | report_path | - | tests/data/report.yaml | - - - Scenario Outline: A partner or redhat associate submits an error-free report - Given the vendor has a valid identity as - And an error-free report is used in - When the user sends a pull request with the report - Then the user sees the pull request is merged - And the index.yaml file is updated with an entry for the submitted chart - - Examples: - | vendor_type | vendor | - | partners | hashicorp | - - Scenario Outline: A community user submits an error-free report - Given the vendor has a valid identity as - And an error-free report is used in - When the user sends a pull request with the report - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | vendor_type | vendor | message | - | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/features/smoke/user_submits_chart_with_errors.feature b/tests/functional/features/smoke/user_submits_chart_with_errors.feature deleted file mode 100644 index a7b03a77ae..0000000000 --- a/tests/functional/features/smoke/user_submits_chart_with_errors.feature +++ /dev/null @@ -1,31 +0,0 @@ -Feature: Chart submission with errors - Partners, redhat or community user submit charts which result in errors - - Examples: - | vendor_type | vendor | chart | version | chart_path | - | partners | hashicorp | vault | 0.17.0 | tests/data/vault-0.17.0.tgz | - - Scenario Outline: An unauthorized user submits a chart - Given A wants to submit a chart in - And of wants to submit of - And the user creates a branch to add a new chart version - When the user sends a pull request with the chart - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | message | user | - | is not allowed to submit the chart on behalf of | unauthorized | - - Scenario Outline: An authorized user submits a chart with incorrect version - Given An authorized user wants to submit a chart in - And of wants to submit of - And Chart.yaml specifies a - And the user creates a branch to add a new chart version - When the user sends a pull request with the chart - Then the pull request is not merged - And user gets the in the pull request comment - - Examples: - | message | bad_version | - | doesn't match the directory structure | 9.9.9 | diff --git a/tests/functional/step_defs/HC-16_test_dash_in_version.py b/tests/functional/step_defs/HC-16_test_dash_in_version.py deleted file mode 100644 index 2cf574f24e..0000000000 --- a/tests/functional/step_defs/HC-16_test_dash_in_version.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -"""Report only submission - -Partners, redhat and community users can publish their chart by submitting -error-free report that was generated by chart-verifier. -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Chart Report Only" - workflow_test = ChartCertificationE2ETestSingle(test_name=test_name) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-16_dash_in_version.feature", - "[HC-16-001] A partner or redhat associate submits report only with dash in chart version", -) -def test_partner_or_redhat_user_submits_report_dash_in_version(): - """A community user submits an error-free report""" diff --git a/tests/functional/step_defs/__init__.py b/tests/functional/step_defs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/functional/step_defs/conftest.py b/tests/functional/step_defs/conftest.py deleted file mode 100644 index 0a82a31c14..0000000000 --- a/tests/functional/step_defs/conftest.py +++ /dev/null @@ -1,264 +0,0 @@ -import pytest -from pytest_bdd import given, then, when, parsers - - -########### GIVEN #################### -@given(parsers.parse("A wants to submit a chart in ")) -def user_wants_to_submit_a_chart(workflow_test, user, chart_path): - """A wants to submit a chart in .""" - workflow_test.update_test_chart(chart_path) - workflow_test.secrets.bot_name = user - - -@given(parsers.parse("An authorized user wants to submit a chart in ")) -def authorized_user_wants_to_submit_a_chart(workflow_test, chart_path): - """A wants to submit a chart in .""" - workflow_test.update_test_chart(chart_path) - - -@given(parsers.parse(" of wants to submit of ")) -def vendor_of_vendor_type_wants_to_submit_chart_of_version( - workflow_test, vendor, vendor_type, chart, version -): - """ of wants to submit of """ - workflow_test.set_vendor(vendor, vendor_type) - workflow_test.chart_name, workflow_test.chart_version = chart, version - - -@given(parsers.parse("Chart.yaml specifies a ")) -def chart_yaml_specifies_bad_version(workflow_test, bad_version): - """Chart.yaml specifies a """ - if bad_version != "": - workflow_test.secrets.bad_version = bad_version - - -@given(parsers.parse("the vendor has a valid identity as ")) -def user_has_valid_identity(workflow_test, vendor, vendor_type): - """the vendor has a valid identity as .""" - workflow_test.set_vendor(vendor, vendor_type) - - -@given( - parsers.parse( - "an error-free chart source is used in and report in " - ) -) -def user_has_created_error_free_chart_src_and_report( - workflow_test, chart_path, report_path -): - """an error-free chart source is used in and report in .""" - workflow_test.update_test_chart(chart_path) - workflow_test.update_test_report(report_path) - - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - workflow_test.process_chart(is_tarball=False) - workflow_test.process_report() - workflow_test.push_chart(is_tarball=False) - - -@given(parsers.parse("an error-free chart source is used in ")) -def user_has_created_error_free_chart_src(workflow_test, chart_path): - """an error-free chart source is used in .""" - workflow_test.update_test_chart(chart_path) - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - workflow_test.process_chart(is_tarball=False) - workflow_test.push_chart(is_tarball=False) - - -@given( - parsers.parse( - "an error-free chart tarball is used in and report in " - ) -) -def user_has_created_error_free_chart_tarball_and_report( - workflow_test, chart_path, report_path -): - """an error-free chart tarball is used in and report in .""" - workflow_test.update_test_chart(chart_path) - workflow_test.update_test_report(report_path) - - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - workflow_test.process_chart(is_tarball=True) - workflow_test.process_report() - workflow_test.push_chart(is_tarball=True) - - -@given(parsers.parse("an error-free chart tarball is used in ")) -def user_has_created_error_free_chart_tarball(workflow_test, chart_path): - """an error-free chart tarball is used in .""" - workflow_test.update_test_chart(chart_path) - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - workflow_test.process_chart(is_tarball=True) - workflow_test.push_chart(is_tarball=True) - - -@given(parsers.parse("report is used in ")) -@given(parsers.parse("an error-free report is used in ")) -def user_has_created_error_free_report(workflow_test, report_path): - """an error-free report is used in .""" - workflow_test.update_test_report(report_path) - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - workflow_test.process_report() - - -@given(parsers.parse("a is provided")) -def user_generated_a_report(workflow_test, report_path): - """report used in """ - workflow_test.update_test_report(report_path) - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - - -@given("the user creates a branch to add a new chart version") -def the_user_creates_a_branch_to_add_a_new_chart_version(workflow_test): - """the user creates a branch to add a new chart version.""" - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - workflow_test.process_chart(is_tarball=False) - if workflow_test.secrets.bad_version: - workflow_test.update_chart_version_in_chart_yaml( - workflow_test.secrets.bad_version - ) - workflow_test.push_chart(is_tarball=False) - - -@given(parsers.parse("chart source is used in ")) -def user_has_used_chart_src(workflow_test, chart_path): - """chart source is used in .""" - workflow_test.update_test_chart(chart_path) - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - workflow_test.process_chart(is_tarball=False) - - -@given( - parsers.parse("a chart tarball is used in and report in ") -) -def user_has_created_a_chart_tarball_and_report(workflow_test, chart_path, report_path): - """an error-free chart tarball is used in and report in .""" - workflow_test.update_test_chart(chart_path) - workflow_test.update_test_report(report_path) - - workflow_test.setup_git_context() - workflow_test.setup_gh_pages_branch() - workflow_test.setup_temp_dir() - workflow_test.process_owners_file() - workflow_test.process_chart(is_tarball=True) - - -@given("README file is missing in the chart") -def readme_file_is_missing(workflow_test): - """README file is missing in the chart""" - workflow_test.remove_readme_file() - - -@given(parsers.parse("the report contains an ")) -def sha_value_does_not_match(workflow_test, invalid_url): - workflow_test.process_report(update_url=True, url=invalid_url) - - -@given("user adds a non chart related file") -def user_adds_a_non_chart_related_file(workflow_test): - """user adds a non chart related file""" - workflow_test.add_non_chart_related_file() - - -@given(parsers.parse("the report contains an ")) -def sha_value_does_not_match(workflow_test, error): - if error == "sha_mismatch": - workflow_test.process_report(update_chart_sha=True) - else: - pytest.fail(f"This {error} handling is not implemented yet") - - -############### WHEN #################### -@when("the user sends a pull request with the report") -@when("the user sends a pull request with the chart") -@when("the user sends a pull request with the chart and report") -def user_sends_pull_request_with_chart_src_and_report(workflow_test): - """the user sends a pull request with the chart and report.""" - workflow_test.send_pull_request() - - -@when("the user pushed the chart and created pull request") -def user_pushed_the_chart_and_created_pull_request_with_chart_src(workflow_test): - """the user pushed the chart and created pull request""" - workflow_test.push_chart(is_tarball=False) - workflow_test.send_pull_request() - - -@when("the user sends a pull request with both chart and non related file") -def user_sends_pull_request_with_chart_and_non_related_file(workflow_test): - """the user sends a pull request with both chart and non related file""" - workflow_test.push_chart(is_tarball=False, add_non_chart_file=True) - workflow_test.send_pull_request() - - -@when("the user sends a pull request with the chart tar and report") -def user_sends_pull_request_with_chart_tarball_and_report(workflow_test): - """the user sends a pull request with the chart and report.""" - workflow_test.push_chart(is_tarball=True) - workflow_test.send_pull_request() - - -################ THEN ################ -@then("the user sees the pull request is merged") -def user_should_see_pull_request_getting_merged(workflow_test): - """the user sees the pull request is merged.""" - workflow_test.check_workflow_conclusion(expect_result="success") - workflow_test.check_pull_request_result(expect_merged=True) - workflow_test.check_pull_request_labels() - - -@then("the pull request is not merged") -def the_pull_request_is_not_getting_merged(workflow_test): - """the pull request is not merged""" - workflow_test.check_workflow_conclusion(expect_result="failure") - workflow_test.check_pull_request_result(expect_merged=False) - - -@then("the index.yaml file is updated with an entry for the submitted chart") -def index_yaml_is_updated_with_new_entry(workflow_test): - """the index.yaml file is updated with an entry for the submitted chart.""" - workflow_test.check_index_yaml() - - -@then( - "the index.yaml file is updated with an entry for the submitted chart with correct providerType" -) -def index_yaml_is_updated_with_new_entry_with_correct_provider_type(workflow_test): - """the index.yaml file is updated with an entry for the submitted chart with correct providerType""" - workflow_test.check_index_yaml(check_provider_type=True) - - -@then("a release is published with corresponding report and chart tarball") -def release_is_published(workflow_test): - """a release is published with corresponding report and chart tarball.""" - workflow_test.check_release_result() - - -@then(parsers.parse("user gets the in the pull request comment")) -def user_gets_the_message_in_the_pull_request_comment(workflow_test, message): - """user gets the message in the pull request comment""" - workflow_test.check_pull_request_comments(expect_message=message) diff --git a/tests/functional/step_defs/test_chart_src_with_report.py b/tests/functional/step_defs/test_chart_src_with_report.py deleted file mode 100644 index 450d2bb4db..0000000000 --- a/tests/functional/step_defs/test_chart_src_with_report.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -"""Chart source submission with report - -Partners, redhat and community users can publish their chart by submitting -error-free chart in source format with a report. -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Chart Source With Report" - test_chart = "tests/data/vault-0.17.0.tgz" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-07_report_and_chart_src.feature", - "[HC-07-001] A partner or redhat associate submits an error-free chart source with report", -) -def test_partner_or_redhat_user_submits_chart_src_with_report(): - """A partner or redhat associate submits an error-free chart source with report.""" - - -@scenario( - "../features/HC-07_report_and_chart_src.feature", - "[HC-07-002] A community user submits an error-free chart source with report", -) -def test_community_user_submits_chart_src_with_report(): - """A community user submits an error-free chart source with report""" diff --git a/tests/functional/step_defs/test_chart_src_without_report.py b/tests/functional/step_defs/test_chart_src_without_report.py deleted file mode 100644 index efa930a28e..0000000000 --- a/tests/functional/step_defs/test_chart_src_without_report.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -"""Chart source submission without report - -Partners, redhat and community users can publish their chart by submitting -error-free chart in source format without a report. -""" -import pytest -from pytest_bdd import scenario - - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Chart Source Without Report" - test_chart = "tests/data/vault-0.17.0.tgz" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-01_chart_src_without_report.feature", - "[HC-01-001] A partner or redhat associate submits an error-free chart source", -) -def test_partner_or_redhat_user_submits_chart_src(): - """A partner or redhat associate submits an error-free chart source.""" - - -@scenario( - "../features/HC-01_chart_src_without_report.feature", - "[HC-01-002] A community user submits an error-free chart source without report", -) -def test_community_user_submits_chart_src(): - """A community user submits an error-free chart source without report""" diff --git a/tests/functional/step_defs/test_chart_tar_with_report.py b/tests/functional/step_defs/test_chart_tar_with_report.py deleted file mode 100644 index e19b2df039..0000000000 --- a/tests/functional/step_defs/test_chart_tar_with_report.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -"""Chart tarball submission with report - -Partners, redhat and community users can publish their chart by submitting -error-free chart in tarball format with a report. -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Chart Tarball With Report" - test_chart = "tests/data/vault-0.17.0.tgz" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-08_report_and_chart_tar.feature", - "[HC-08-001] A partner or redhat associate submits an error-free chart tarball with report", -) -def test_partners_or_redhat_user_submits_chart_tarball_with_report(): - """A partner or redhat associate submits an error-free chart tarball with report.""" - - -@scenario( - "../features/HC-08_report_and_chart_tar.feature", - "[HC-08-002] A community user submits an error-free chart tarball with report", -) -def test_community_user_submits_chart_tarball_with_report(): - """A community user submits an error-free chart tarball with report""" diff --git a/tests/functional/step_defs/test_chart_tar_without_report.py b/tests/functional/step_defs/test_chart_tar_without_report.py deleted file mode 100644 index abf8efe33a..0000000000 --- a/tests/functional/step_defs/test_chart_tar_without_report.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -"""Chart tarball submission without report - -Partners, redhat and community users can publish their chart by submitting -error-free chart in tarball format without a report. -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Chart Tarball Without Report" - test_chart = "tests/data/vault-0.17.0.tgz" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-02_chart_tar_without_report.feature", - "[HC-02-001] A partner or redhat associate submits an error-free chart tarball", -) -def test_partner_or_redhat_user_submits_chart_tarball(): - """A partner or redhat associate submits an error-free chart tarball.""" - - -@scenario( - "../features/HC-02_chart_tar_without_report.feature", - "[HC-02-002] A community user submits an error-free chart tarball without report", -) -def test_community_user_submits_chart_tarball(): - """A community user submits an error-free chart tarball without report""" diff --git a/tests/functional/step_defs/test_chart_test_takes_more_than_30mins.py b/tests/functional/step_defs/test_chart_test_takes_more_than_30mins.py deleted file mode 100644 index a38356cac8..0000000000 --- a/tests/functional/step_defs/test_chart_test_takes_more_than_30mins.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -""" Chart test takes longer time and exceeds default timeout - Partners, redhat or community user submit charts which result in errors -""" -import datetime -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Chart test takes more than 30mins" - test_chart = "tests/data/vault-test-timeout-0.17.0.tgz" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart - ) - start_time = datetime.datetime.now() - yield workflow_test - workflow_test.cleanup() - end_time = datetime.datetime.now() - time_diff = end_time - start_time - total_diff_seconds = time_diff.total_seconds() - if not int(total_diff_seconds) >= 1800: - pytest.fail(f"Timeout is not as expected: {total_diff_seconds}") - - -@scenario( - "../features/HC-16_chart_test_takes_more_than_30mins.feature", - "[HC-16-001] A partner or community user submits chart that takes more than 30 mins", -) -def test_partner_or_community_chart_test_takes_more_than_30mins(): - """A partner or community submitted chart takes more than 30 mins""" - - -@scenario( - "../features/HC-16_chart_test_takes_more_than_30mins.feature", - "[HC-16-002] A redhat associate submits a chart that takes more than 30 mins", -) -def test_redhat_chart_test_takes_more_than_30mins(): - """A redhat submitted chart takes more than 30 mins""" diff --git a/tests/functional/step_defs/test_chart_verifier_comes_back_with_failures.py b/tests/functional/step_defs/test_chart_verifier_comes_back_with_failures.py deleted file mode 100644 index 7529f0f6cc..0000000000 --- a/tests/functional/step_defs/test_chart_verifier_comes_back_with_failures.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Chart verifier comes back with a failure -Partners, redhat or community user submit charts which does not contain README file -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Chart Submission Without Readme" - test_chart = "tests/data/vault-0.17.0.tgz" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-03_chart_verifier_comes_back_with_failures.feature", - "[HC-03-001] A partner or community user submits a chart which does not contain a readme file", -) -def test_partner_or_community_user_submits_chart_without_readme(): - """A partner or community user submits a chart which does not contain a readme file""" - - -@scenario( - "../features/HC-03_chart_verifier_comes_back_with_failures.feature", - "[HC-03-002] A redhat user submits a chart which does not contain a readme file", -) -def test_redhat_user_submits_chart_without_readme(): - """A redhat user submits a chart which does not contain a readme file""" diff --git a/tests/functional/step_defs/test_invalid_url_in_the_report.py b/tests/functional/step_defs/test_invalid_url_in_the_report.py deleted file mode 100644 index d5691c5d4d..0000000000 --- a/tests/functional/step_defs/test_invalid_url_in_the_report.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Report contains an invalid URL -Partners, redhat and community users submits only report with an invalid URL -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Invalid Chart URL" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-04_invalid_url_in_the_report.feature", - "[HC-04-001] A user submits a report with an invalid url", -) -def test_report_submission_with_invalid_url(): - """A user submits a report with an invalid url.""" diff --git a/tests/functional/step_defs/test_pr_includes_a_file_which_is_not_related.py b/tests/functional/step_defs/test_pr_includes_a_file_which_is_not_related.py deleted file mode 100644 index cf64b378ef..0000000000 --- a/tests/functional/step_defs/test_pr_includes_a_file_which_is_not_related.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -""" -PR includes a non chart related file -Partners, redhat or community user submit charts which includes a file which is not part of the chart -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test PR Includes A Non Related File" - test_chart = "tests/data/vault-0.17.0.tgz" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-05_pr_includes_a_file_which_is_not_chart_related.feature", - "[HC-05-001] A user submits a chart with non chart related file", -) -def test_user_submits_chart_with_non_related_file(): - """A user submits a chart with non chart related file""" diff --git a/tests/functional/step_defs/test_provider_delivery_control.py b/tests/functional/step_defs/test_provider_delivery_control.py deleted file mode 100644 index 383c8d1ba0..0000000000 --- a/tests/functional/step_defs/test_provider_delivery_control.py +++ /dev/null @@ -1,114 +0,0 @@ -# -*- coding: utf-8 -*- -"""Chart tarball submission with report - -Partners, redhat and community users can publish their chart by submitting -error-free chart in tarball format with a report. -""" -import pytest -from pytest_bdd import scenario, given, parsers, then -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Porvider Delivery Control" - test_chart = "tests/data/vault-0.17.0.tgz" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-06_provider_delivery_control.feature", - "[HC-06-001] A partner associate submits an error-free report with provider controlled delivery", -) -def test_partners_submits_error_free_report_for_provider_controlled_delivery(): - """A partner submits an error-free report for provider controlled delivery.""" - - -@scenario( - "../features/HC-06_provider_delivery_control.feature", - "[HC-06-002] A partner associate submits an error-free report and chart with provider controlled delivery", -) -def test_partners_submits_error_free_report_and_chart_for_provider_controlled_delivery(): - """A partner submits an error-free report and chart for provider controlled delivery.""" - - -@scenario( - "../features/HC-06_provider_delivery_control.feature", - "[HC-06-003] A partner associate submits an error-free report with inconsistent provider controlled delivery setting", -) -def test_partners_submits_error_free_report_with_inconsistent_provider_controlled_delivery_settings(): - """A partner submits an error-free report with inconsistent settings for provider controlled delivery.""" - - -@given( - parsers.parse( - "provider delivery control is set to in the OWNERS file" - ) -) -def provider_delivery_control_set_in_owners(workflow_test, provider_control_owners): - if provider_control_owners == "true": - print("[INFO] set provider delivery control_in owners file") - workflow_test.secrets.provider_delivery = True - else: - print("[INFO] un-set provider delivery control_in owners file") - workflow_test.secrets.provider_delivery = False - - -@given( - parsers.parse( - "provider delivery control is set to in the report" - ) -) -def provider_delivery_control_set_in_report(workflow_test, provider_control_report): - if provider_control_report == "true": - print("[INFO] set provider delivery control_in report") - workflow_test.process_report( - update_provider_delivery=True, provider_delivery=True - ) - else: - print("[INFO] un-set provider delivery control_in report") - workflow_test.process_report( - update_provider_delivery=True, provider_delivery=False - ) - - -@given( - parsers.parse( - "provider delivery control is set to and a package digest is in the report" - ) -) -def provider_delivery_control_and_package_digest_set_in_report( - workflow_test, provider_control_report, package_digest_set=True -): - if package_digest_set == "true": - no_package_digest = False - else: - no_package_digest = True - - if provider_control_report == "true": - print("[INFO] set provider delivery control_in report") - workflow_test.process_report( - update_provider_delivery=True, - provider_delivery=True, - unset_package_digest=no_package_digest, - ) - else: - print("[INFO] un-set provider delivery control_in report") - workflow_test.process_report( - update_provider_delivery=True, - provider_delivery=False, - unset_package_digest=no_package_digest, - ) - - -@then( - parsers.parse("the is updated with an entry for the submitted chart") -) -def index_file_is_updated(workflow_test, index_file): - workflow_test.secrets.index_file = index_file - workflow_test.check_index_yaml(True) diff --git a/tests/functional/step_defs/test_report_in_json_format.py b/tests/functional/step_defs/test_report_in_json_format.py deleted file mode 100644 index 3bf59b64b6..0000000000 --- a/tests/functional/step_defs/test_report_in_json_format.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Report only submission in json format -Partners, redhat and community users trying to publish chart by submitting report in json format -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Report in Json Format" - test_report = "tests/data/report.json" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-09_report_in_json_format.feature", - "[HC-09-001] An user submits an report in json format", -) -def an_user_submits_report_in_json_format(): - """An user submits an report in json format.""" diff --git a/tests/functional/step_defs/test_report_only_edited.py b/tests/functional/step_defs/test_report_only_edited.py deleted file mode 100644 index 684eca9a04..0000000000 --- a/tests/functional/step_defs/test_report_only_edited.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest -from pytest_bdd import scenario, given, parsers - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Edited Report Failures" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-10_report_only_edited.feature", - "[HC-10-001] A partner or redhat associate submits an edited report", -) -def test_partner_or_redhat_user_submits_edited_report(): - """A partner or redhat associate submits an edited report.""" - - -@given( - parsers.parse( - "the report includes and OpenshiftVersion values and chart value" - ) -) -def report_includes_specified_versions(workflow_test, tested, supported, kubeversion): - workflow_test.process_report( - update_versions=True, - supported_versions=supported, - tested_version=tested, - kube_version=kubeversion, - ) diff --git a/tests/functional/step_defs/test_report_only_no_errors.py b/tests/functional/step_defs/test_report_only_no_errors.py deleted file mode 100644 index a77e5ae707..0000000000 --- a/tests/functional/step_defs/test_report_only_no_errors.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -"""Report only submission - -Partners, redhat and community users can publish their chart by submitting -error-free report that was generated by chart-verifier. -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Test Chart Report Only" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-12_report_without_chart.feature", - "[HC-12-001] A partner or redhat associate submits an error-free report", -) -def test_partner_or_redhat_user_submits_report(): - """A partner or redhat associate submits an error-free report.""" - - -@scenario( - "../features/HC-12_report_without_chart.feature", - "[HC-12-002] A community user submits an error-free report", -) -def test_community_user_submits_report(): - """A community user submits an error-free report""" diff --git a/tests/functional/step_defs/test_report_with_missing_checks.py b/tests/functional/step_defs/test_report_with_missing_checks.py deleted file mode 100644 index 5e3fa6b972..0000000000 --- a/tests/functional/step_defs/test_report_with_missing_checks.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Report does not include a check -Partners, redhat and community users submits only report which does not include full set of checks -""" -import pytest -from pytest_bdd import scenario, given, parsers - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Report with missing checks" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-11_report_with_missing_checks.feature", - "[HC-11-001] A user submits a report with missing checks", -) -def test_report_submission_with_missing_checks(): - """A user submits a report with missing checks.""" - - -@given(parsers.parse("the report has a missing")) -def report_has_a_check_missing(workflow_test, check): - workflow_test.process_report(missing_check=check) diff --git a/tests/functional/step_defs/test_sha_value_does_not_match.py b/tests/functional/step_defs/test_sha_value_does_not_match.py deleted file mode 100644 index 373dc22738..0000000000 --- a/tests/functional/step_defs/test_sha_value_does_not_match.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -"""SHA value in the report does not match -Partners, redhat and community users submits chart tar with report -where tar sha does not match with sha value digests.chart in the report -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "SHA Value Does Not Match" - test_chart = "tests/data/vault-0.17.0.tgz" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-13_sha_value_does_not_match.feature", - "[HC-13-001] A user submits a chart tarball with report", -) -def test_chart_submission_with_report(): - """A user submits a chart tarball with report.""" diff --git a/tests/functional/step_defs/test_smoke_scenarios.py b/tests/functional/step_defs/test_smoke_scenarios.py deleted file mode 100644 index 12dbbc2aca..0000000000 --- a/tests/functional/step_defs/test_smoke_scenarios.py +++ /dev/null @@ -1,203 +0,0 @@ -import pytest -from pytest_bdd import scenario, given, parsers, then - - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Smoke Test" - test_chart = "tests/data/vault-0.17.0.tgz" - test_report = "tests/data/report.yaml" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart, test_report=test_report - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/smoke/chart_src_without_report.feature", - "A partner or redhat associate submits an error-free chart source", -) -def test_partner_or_redhat_user_submits_chart_src(): - """A partner or redhat associate submits an error-free chart source.""" - - -@scenario( - "../features/smoke/chart_tar_without_report.feature", - "A partner or redhat associate submits an error-free chart tarball", -) -def test_partner_or_redhat_user_submits_chart_tarball(): - """A partner or redhat associate submits an error-free chart tarball.""" - - -@scenario( - "../features/smoke/report_and_chart_src.feature", - "A partner or redhat associate submits an error-free chart source with report", -) -def test_partner_or_redhat_user_submits_chart_src_with_report(): - """A partner or redhat associate submits an error-free chart source with report.""" - - -@scenario( - "../features/smoke/report_and_chart_src.feature", - "A community user submits an error-free chart source with report", -) -def test_community_user_submits_chart_src_with_report(): - """A community user submits an error-free chart source with report""" - - -@scenario( - "../features/smoke/report_and_chart_tar.feature", - "A partner or redhat associate submits an error-free chart tarball with report", -) -def test_partners_or_redhat_user_submits_chart_tarball_with_report(): - """A partner or redhat associate submits an error-free chart tarball with report.""" - - -@scenario( - "../features/smoke/report_and_chart_tar.feature", - "A community user submits an error-free chart tarball with report", -) -def test_community_user_submits_chart_tarball_with_report(): - """A community user submits an error-free chart tarball with report""" - - -@scenario( - "../features/smoke/report_without_chart.feature", - "A partner or redhat associate submits an error-free report", -) -def test_partner_or_redhat_user_submits_report(): - """A partner or redhat associate submits an error-free report.""" - - -@scenario( - "../features/smoke/report_without_chart.feature", - "A community user submits an error-free report", -) -def test_community_user_submits_report(): - """A community user submits an error-free report""" - - -@scenario( - "../features/smoke/chart_verifier_comes_back_with_failures.feature", - "A partner or community user submits a chart which does not contain a readme file", -) -def test_partner_or_community_user_submits_chart_without_readme(): - """A partner or community user submits a chart which does not contain a readme file""" - - -@scenario( - "../features/smoke/invalid_url_in_the_report.feature", - "A user submits a report with an invalid url", -) -def test_report_submission_with_invalid_url(): - """A user submits a report with an invalid url.""" - - -@scenario( - "../features/smoke/pr_includes_a_file_which_is_not_chart_related.feature", - "A user submits a chart with non chart related file", -) -def test_user_submits_chart_with_non_related_file(): - """A user submits a chart with non chart related file""" - - -@scenario( - "../features/smoke/report_only_edited.feature", - "A partner or redhat associate submits an edited report", -) -def test_partner_or_redhat_user_submits_edited_report(): - """A partner or redhat associate submits an edited report.""" - - -@scenario( - "../features/smoke/report_with_missing_checks.feature", - "A user submits a report with missing checks", -) -def test_report_submission_with_missing_checks(): - """A user submits a report with missing checks.""" - - -@scenario( - "../features/smoke/user_submits_chart_with_errors.feature", - "An unauthorized user submits a chart", -) -def test_chart_submission_by_unauthorized_user(): - """An unauthorized user submits a chart""" - - -@scenario( - "../features/smoke/user_submits_chart_with_errors.feature", - "An authorized user submits a chart with incorrect version", -) -def test_chart_submission_with_incorrect_version(): - """An authorized user submits a chart with incorrect version""" - - -@scenario( - "../features/smoke/provider_delivery_control.feature", - "A partner associate submits an error-free report with provider controlled delivery", -) -def test_partners_submits_error_free_report_for_provider_controlled_delivery(): - """A partner submits an error-free report for provider controlled delivery.""" - - -@given(parsers.parse("the report has a missing")) -def report_has_a_check_missing(workflow_test, check): - workflow_test.process_report(missing_check=check) - - -@given( - parsers.parse( - "the report includes and OpenshiftVersion values and chart value" - ) -) -def report_includes_specified_versions(workflow_test, tested, supported, kubeversion): - workflow_test.process_report( - update_versions=True, - supported_versions=supported, - tested_version=tested, - kube_version=kubeversion, - ) - - -@given( - parsers.parse( - "provider delivery control is set to in the OWNERS file" - ) -) -def provider_delivery_control_set_in_owners(workflow_test, provider_control_owners): - if provider_control_owners == "true": - print("[INFO] set provider delivery control_in owners file") - workflow_test.secrets.provider_delivery = True - else: - print("[INFO] un-set provider delivery control_in owners file") - workflow_test.secrets.provider_delivery = False - - -@given( - parsers.parse( - "provider delivery control is set to in the report" - ) -) -def provider_delivery_control_set_in_report(workflow_test, provider_control_report): - if provider_control_report == "true": - print("[INFO] set provider delivery control_in report") - workflow_test.process_report( - update_provider_delivery=True, provider_delivery=True - ) - else: - print("[INFO] un-set provider delivery control_in report") - workflow_test.process_report( - update_provider_delivery=True, provider_delivery=False - ) - - -@then( - parsers.parse("the is updated with an entry for the submitted chart") -) -def index_file_is_updated(workflow_test, index_file): - workflow_test.secrets.index_file = index_file diff --git a/tests/functional/step_defs/test_submitted_charts.py b/tests/functional/step_defs/test_submitted_charts.py deleted file mode 100644 index 3e6aadd0b9..0000000000 --- a/tests/functional/step_defs/test_submitted_charts.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -"""Check submitted charts - -New Openshift or chart-verifier will trigger (automatically or manually) a recursive checking on -existing submitted charts under `charts/` directory with the specified Openshift and chart-verifier -version. - -Besides, during workflow development, engineers would like to check if the changes will break checks -on existing submitted charts. -""" -import pytest -from pytest_bdd import ( - given, - scenario, - then, - when, -) -from functional.utils.chart_certification import ChartCertificationE2ETestMultiple - - -@pytest.fixture -def workflow_test(): - workflow_test = ChartCertificationE2ETestMultiple() - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-15_check_submitted_charts.feature", - "[HC-15-001] A new Openshift or chart-verifier version is specified either by a cron job or manually", -) -def test_submitted_charts(): - """A new Openshift or chart-verifier version is specified either by a cron job or manually.""" - - -@given("there's a github workflow for testing existing charts") -def theres_github_workflow_for_testing_charts(): - """there's a github workflow for testing existing charts.""" - - -@when("a new Openshift or chart-verifier version is specified") -def new_openshift_or_verifier_version_is_specified(): - """a new Openshift or chart-verifier version is specified.""" - - -@when("the vendor type is specified, e.g. partner, and/or redhat") -def vendor_type_is_specified(): - """the vendor type is specified, e.g. partner, and/or redhat.""" - - -@when("workflow for testing existing charts is triggered") -def workflow_is_triggered(): - """workflow for testing existing charts is triggered.""" - - -@then("submission tests are run for existing charts") -def submission_tests_run_for_submitted_charts(workflow_test): - """submission tests are run for existing charts.""" - workflow_test.process_all_charts() - - -@then("all results are reported back to the caller") -def all_results_report_back_to_caller(): - """all results are reported back to the caller.""" diff --git a/tests/functional/step_defs/test_user_submits_chart_with_errors.py b/tests/functional/step_defs/test_user_submits_chart_with_errors.py deleted file mode 100644 index 4d7d4cf31d..0000000000 --- a/tests/functional/step_defs/test_user_submits_chart_with_errors.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -""" Chart submission with errors - Partners, redhat or community user submit charts which result in errors -""" -import pytest -from pytest_bdd import scenario - -from functional.utils.chart_certification import ChartCertificationE2ETestSingle - - -@pytest.fixture -def workflow_test(): - test_name = "Chart Submission with Errors" - test_chart = "tests/data/vault-0.17.0.tgz" - workflow_test = ChartCertificationE2ETestSingle( - test_name=test_name, test_chart=test_chart - ) - yield workflow_test - workflow_test.cleanup() - - -@scenario( - "../features/HC-14_user_submits_chart_with_errors.feature", - "[HC-14-001] An unauthorized user submits a chart", -) -def test_chart_submission_by_unauthorized_user(): - """An unauthorized user submits a chart""" - - -@scenario( - "../features/HC-14_user_submits_chart_with_errors.feature", - "[HC-14-002] An authorized user submits a chart with incorrect version", -) -def test_chart_submission_with_incorrect_version(): - """An authorized user submits a chart with incorrect version""" diff --git a/tests/functional/utils/__init__.py b/tests/functional/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/functional/utils/chart.py b/tests/functional/utils/chart.py deleted file mode 100644 index c565672156..0000000000 --- a/tests/functional/utils/chart.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding: utf-8 -*- -"""Utility module for processing chart files.""" - -import os -import tarfile -import pytest -import yaml -import shutil - - -def get_name_and_version_from_report(path): - """ - Parameters: - path (str): path to the report.yaml - - Returns: - str: chart name - str: chart version - """ - with open(path, "r") as fd: - try: - report = yaml.safe_load(fd) - except yaml.YAMLError as err: - pytest.fail(f"error parsing '{path}': {err}") - chart = report["metadata"]["chart"] - return chart["name"], chart["version"] - - -def get_name_and_version_from_chart_tar(path): - """ - Parameters: - path (str): path to the chart tar file - - Returns: - str: chart name - str: chart version - """ - tar = tarfile.open(path) - for member in tar.getmembers(): - if member.name.split("/")[-1] == "Chart.yaml": - chart = tar.extractfile(member) - if chart is not None: - content = chart.read() - try: - chart_yaml = yaml.safe_load(content) - return chart_yaml["name"], chart_yaml["version"] - except yaml.YAMLError as err: - pytest.fail(f"error parsing '{path}': {err}") - else: - pytest.fail(f"Chart.yaml not in {path}") - - -def get_name_and_version_from_chart_src(path): - """ - Parameters: - path (str): path to the chart src directory - - Returns: - str: chart name - str: chart version - """ - chart_path = os.path.join(path, "Chart.yaml") - with open(chart_path, "r") as fd: - try: - chart_yaml = yaml.safe_load(fd) - except yaml.YAMLError as err: - pytest.fail(f"error parsing '{path}': {err}") - return chart_yaml["name"], chart_yaml["version"] - - -def extract_chart_tgz(src, dst, secrets, logger): - """Extracts the chart tgz file into the target location under 'charts/' for PR submission tests - - Parameters: - src (str): path to the test chart tgz - dst (str): path to the extract destination, e.g. 'charts/partners/hashicorp/vault/0.13.0' - """ - try: - logger.info(f"Remove existing local '{dst}/src'") - shutil.rmtree(f"{dst}/src") - except FileNotFoundError: - logger.info(f"'{dst}/src' does not exist") - finally: - with tarfile.open(src, "r") as fd: - fd.extractall(dst) - os.rename(f"{dst}/{secrets.chart_name}", f"{dst}/src") - - -def get_all_charts(charts_path: str, vendor_types: str) -> list: - # TODO: Support `community` as vendor_type. - """Gets charts with src or tgz under `charts/` given vendor_types and without report. - - Parameters: - charts_path (str): path to the `charts/` directory - vendor_types (str): vendor type to look for, any combination of `partner`, `redhat`, separated - by commas, or `all` to run both `partner` and `redhat`. - - Returns: - list: list of (vendor_type, vendor, chart_name, chart_version) tuples - """ - ret = [] - # Pre-process vendor types - vendor_types = vendor_types.replace("partner", "partners") - vendor_types = [vt.strip() for vt in vendor_types.split(",")] - vendor_types = list({"partners", "redhat", "all"}.intersection(set(vendor_types))) - vendor_types = ["partners", "redhat"] if "all" in vendor_types else vendor_types - - # Iterate through `charts/` to find chart submission with src or tgz - for vt in vendor_types: - charts_path_vt = f"{charts_path}/{vt}" - vendor_names = [ - name - for name in os.listdir(charts_path_vt) - if os.path.isdir(f"{charts_path_vt}/{name}") - ] - for vn in vendor_names: - charts_path_vt_vn = f"{charts_path_vt}/{vn}" - chart_names = [ - name - for name in os.listdir(charts_path_vt_vn) - if os.path.isdir(f"{charts_path_vt_vn}/{name}") - ] - for cn in chart_names: - charts_path_vt_vn_cn = f"{charts_path_vt_vn}/{cn}" - file_names = [name for name in os.listdir(charts_path_vt_vn_cn)] - if "OWNERS" not in file_names: - continue - chart_versions = [ - name - for name in os.listdir(charts_path_vt_vn_cn) - if os.path.isdir(f"{charts_path_vt_vn_cn}/{name}") - ] - # Only interest in latest chart version - if len(chart_versions) == 0: - continue - cv = max(chart_versions) - charts_path_vt_vn_cn_cv = f"{charts_path_vt_vn_cn}/{cv}" - file_names = [name for name in os.listdir(charts_path_vt_vn_cn_cv)] - if "report.yaml" not in file_names and ( - f"{cn}-{cv}.tgz" in file_names or "src" in file_names - ): - ret.append((vt, vn, cn, cv)) - return ret diff --git a/tests/functional/utils/chart_certification.py b/tests/functional/utils/chart_certification.py deleted file mode 100644 index 920b241036..0000000000 --- a/tests/functional/utils/chart_certification.py +++ /dev/null @@ -1,1347 +0,0 @@ -# -*- coding: utf-8 -*- -"""Utility class for setting up and manipulating certification workflow tests.""" - -import os -import re -import json -import pathlib -import shutil -import logging -import time -import uuid -from tempfile import TemporaryDirectory -from dataclasses import dataclass -from string import Template -from pathlib import Path - -import git -import yaml -import pytest -from functional.utils.notifier import * -from functional.utils.index import * -from functional.utils.github import * -from functional.utils.secret import * -from functional.utils.set_directory import SetDirectory -from functional.utils.setttings import * -from functional.utils.chart import * - - -@dataclass -class ChartCertificationE2ETest: - owners_file_content: str = """\ -chart: - name: ${chart_name} - shortDescription: Test chart for testing chart submission workflows. -publicPgpKey: null -providerDelivery: ${provider_delivery} -users: -- githubUsername: ${bot_name} -vendor: - label: ${vendor} - name: ${vendor} -""" - secrets: E2ETestSecret = E2ETestSecret() - - old_cwd: str = os.getcwd() - repo: git.Repo = git.Repo() - temp_dir: TemporaryDirectory = None - temp_repo: git.Repo = None - github_actions: str = os.environ.get("GITHUB_ACTIONS") - - def set_git_username_email(self, repo, username, email): - """ - Parameters: - repo (git.Repo): git.Repo instance of the local directory - username (str): git username to set - email (str): git email to set - """ - repo.config_writer().set_value("user", "name", username).release() - repo.config_writer().set_value("user", "email", email).release() - - def get_bot_name_and_token(self): - bot_name = os.environ.get("BOT_NAME") - bot_token = os.environ.get("BOT_TOKEN") - if not bot_name and not bot_token: - bot_name = "github-actions[bot]" - bot_token = os.environ.get("GITHUB_TOKEN") - if not bot_token: - raise Exception("BOT_TOKEN environment variable not defined") - elif not bot_name: - raise Exception("BOT_TOKEN set but BOT_NAME not specified") - elif not bot_token: - raise Exception("BOT_NAME set but BOT_TOKEN not specified") - return bot_name, bot_token - - def remove_chart( - self, chart_directory, chart_version, remote_repo, base_branch, bot_token - ): - # Remove chart files from base branch - logging.info( - f"Remove {chart_directory}/{chart_version} from {remote_repo}:{base_branch}" - ) - try: - self.temp_repo.git.rm( - "-rf", "--cached", f"{chart_directory}/{chart_version}" - ) - self.temp_repo.git.commit("-m", f"Remove {chart_directory}/{chart_version}") - self.temp_repo.git.push( - f"https://x-access-token:{bot_token}@github.com/{remote_repo}", - f"HEAD:refs/heads/{base_branch}", - ) - except git.exc.GitCommandError: - logging.info( - f"{chart_directory}/{chart_version} not exist on {remote_repo}:{base_branch}" - ) - - def remove_owners_file(self, chart_directory, remote_repo, base_branch, bot_token): - # Remove the OWNERS file from base branch - logging.info( - f"Remove {chart_directory}/OWNERS from {remote_repo}:{base_branch}" - ) - try: - self.temp_repo.git.rm("-rf", "--cached", f"{chart_directory}/OWNERS") - self.temp_repo.git.commit("-m", f"Remove {chart_directory}/OWNERS") - self.temp_repo.git.push( - f"https://x-access-token:{bot_token}@github.com/{remote_repo}", - f"HEAD:refs/heads/{base_branch}", - ) - except git.exc.GitCommandError: - logging.info( - f"{chart_directory}/OWNERS not exist on {remote_repo}:{base_branch}" - ) - - def create_test_gh_pages_branch(self, remote_repo, base_branch, bot_token): - # Get SHA from 'dev-gh-pages' branch - logging.info( - f"Create '{remote_repo}:{base_branch}-gh-pages' from '{remote_repo}:dev-gh-pages'" - ) - r = github_api( - "get", f"repos/{remote_repo}/git/ref/heads/dev-gh-pages", bot_token - ) - j = json.loads(r.text) - sha = j["object"]["sha"] - - # Create a new gh-pages branch for testing - data = {"ref": f"refs/heads/{base_branch}-gh-pages", "sha": sha} - r = github_api("post", f"repos/{remote_repo}/git/refs", bot_token, json=data) - - logging.info(f"gh-pages branch created: {base_branch}-gh-pages") - - def setup_git_context(self, repo: git.Repo): - self.set_git_username_email( - repo, self.secrets.bot_name, GITHUB_ACTIONS_BOT_EMAIL - ) - if os.environ.get("WORKFLOW_DEVELOPMENT"): - logging.info("Wokflow development enabled") - repo.git.add(A=True) - repo.git.commit("-m", "Checkpoint") - - def send_pull_request(self, remote_repo, base_branch, pr_branch, bot_token): - data = { - "head": pr_branch, - "base": base_branch, - "title": base_branch, - "body": os.environ.get("PR_BODY"), - } - - logging.info(f"Create PR from '{remote_repo}:{pr_branch}'") - r = github_api("post", f"repos/{remote_repo}/pulls", bot_token, json=data) - j = json.loads(r.text) - if "number" not in j: - pytest.fail(f"error sending pull request, response was: {r.text}") - return j["number"] - - def create_and_push_owners_file( - self, - chart_directory, - base_branch, - vendor_name, - vendor_type, - chart_name, - provider_delivery=False, - ): - with SetDirectory(Path(self.temp_dir.name)): - # Create the OWNERS file from the string template - values = { - "bot_name": self.secrets.bot_name, - "vendor": vendor_name, - "chart_name": chart_name, - "provider_delivery": provider_delivery, - } - content = Template(self.secrets.owners_file_content).substitute(values) - with open(f"{chart_directory}/OWNERS", "w") as fd: - fd.write(content) - - # Push OWNERS file to the test_repo - logging.info( - f"Push OWNERS file to '{self.secrets.test_repo}:{base_branch}'" - ) - self.temp_repo.git.add(f"{chart_directory}/OWNERS") - self.temp_repo.git.commit( - "-m", f"Add {vendor_type} {vendor_name} {chart_name} OWNERS file" - ) - self.temp_repo.git.push( - f"https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}", - f"HEAD:refs/heads/{base_branch}", - "-f", - ) - - def check_index_yaml( - self, - base_branch, - vendor, - chart_name, - chart_version, - index_file="index.yaml", - check_provider_type=False, - logger=pytest.fail, - ): - old_branch = self.repo.active_branch.name - self.repo.git.fetch( - f"https://github.com/{self.secrets.test_repo}.git", - "{0}:{0}".format(f"{base_branch}-gh-pages"), - "-f", - ) - - self.repo.git.checkout(f"{base_branch}-gh-pages") - - with open(index_file, "r") as fd: - try: - index = yaml.safe_load(fd) - except yaml.YAMLError as err: - logger(f"error parsing index.yaml: {err}") - return False - - if index: - entry = f"{vendor}-{chart_name}" - if "entries" not in index or entry not in index["entries"]: - logger(f"{entry} not added in entries to {index_file}") - logger(f"Index.yaml entries: {index['entries']}") - return False - - version_list = [release["version"] for release in index["entries"][entry]] - if chart_version not in version_list: - logger(f"{chart_version} not added to {index_file}") - logger(f"Index.yaml entry content: {index['entries'][entry]}") - return False - - # This check is applicable for charts submitted in redhat path when one of the chart-verifier check fails - # Check whether providerType annotations is community in index.yaml when vendor_type is redhat - if check_provider_type and self.secrets.vendor_type == "redhat": - provider_type_in_index_yaml = index["entries"][entry][0]["annotations"][ - "charts.openshift.io/providerType" - ] - if provider_type_in_index_yaml != "community": - logger( - f"{provider_type_in_index_yaml} is not correct as providerType in index.yaml" - ) - - logging.info("Index updated correctly, cleaning up local branch") - self.repo.git.checkout(old_branch) - self.repo.git.branch("-D", f"{base_branch}-gh-pages") - return True - else: - return False - - def check_release_result( - self, vendor, chart_name, chart_version, chart_tgz, logger=pytest.fail - ): - expected_tag = f"{vendor}-{chart_name}-{chart_version}" - try: - release = get_release_by_tag(self.secrets, expected_tag) - logging.info(f"Released '{expected_tag}' successfully") - - expected_chart_asset = f"{vendor}-{chart_tgz}" - required_assets = [expected_chart_asset] - logging.info(f"Check '{required_assets}' is in release assets") - release_id = release["id"] - get_release_assets(self.secrets, release_id, required_assets) - return True - except Exception as e: - logger(e) - return False - finally: - logging.info(f"Delete release '{expected_tag}'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/releases/{release_id}", - self.secrets.bot_token, - ) - - logging.info(f"Delete release tag '{expected_tag}'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/tags/{expected_tag}", - self.secrets.bot_token, - ) - - # expect_result: a string representation of expected result, e.g. 'success' - def check_workflow_conclusion( - self, pr_number, expect_result: str, logger=pytest.fail - ): - try: - # Check workflow conclusion - run_id = get_run_id(self.secrets, pr_number) - conclusion = get_run_result(self.secrets, run_id) - if conclusion == expect_result: - logging.info( - f"PR{pr_number} Workflow run was '{expect_result}' which is expected" - ) - else: - logger( - f"PR{pr_number if pr_number else self.secrets.pr_number} Workflow run was '{conclusion}' which is unexpected, run id: {run_id}" - ) - return run_id, conclusion - except Exception as e: - logger(e) - return None, None - - # expect_merged: boolean representing whether the PR should be merged - def check_pull_request_result( - self, pr_number, expect_merged: bool, logger=pytest.fail - ): - # Check if PR merged - r = github_api( - "get", - f"repos/{self.secrets.test_repo}/pulls/{pr_number}/merge", - self.secrets.bot_token, - ) - logging.info(f"PR{pr_number} result status_code : {r.status_code}") - if r.status_code == 204 and expect_merged: - logging.info(f"PR{pr_number} merged sucessfully as expected") - return True - elif r.status_code == 404 and not expect_merged: - logging.info(f"PR{pr_number} not merged, which is expected") - return True - elif r.status_code == 204 and not expect_merged: - logger(f"PR{pr_number} Expecting not merged but PR was merged") - return False - elif r.status_code == 404 and expect_merged: - logger(f"PR{pr_number} Expecting PR merged but PR was not merged") - return False - else: - logger(f"PR{pr_number} Got unexpected status code from PR: {r.status_code}") - return False - - def check_pull_request_labels(self, pr_number, logger=pytest.fail): - r = github_api( - "get", - f"repos/{self.secrets.test_repo}/issues/{pr_number}/labels", - self.secrets.bot_token, - ) - labels = json.loads(r.text) - authorized_request = False - content_ok = False - for label in labels: - logging.info(f"PR{pr_number} found label {label['name']}") - if label["name"] == "authorized-request": - authorized_request = True - if label["name"] == "content-ok": - content_ok = True - - if authorized_request and content_ok: - logging.info( - f"PR{pr_number} authorized request and content-ok labels were found as expected" - ) - return True - else: - logger( - f"PR{pr_number} authorized request and/or content-ok labels were not found as expected" - ) - return False - - def cleanup_release(self, expected_tag): - """Cleanup the release and release tag. - - Releases might be left behind if check_index_yam() ran before check_release_result() and fails the test. - """ - r = github_api( - "get", f"repos/{self.secrets.test_repo}/releases", self.secrets.bot_token - ) - releases = json.loads(r.text) - for release in releases: - if release["tag_name"] == expected_tag: - release_id = release["id"] - logging.info(f"Delete release '{expected_tag}'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/releases/{release_id}", - self.secrets.bot_token, - ) - - logging.info(f"Delete release tag '{expected_tag}'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/tags/{expected_tag}", - self.secrets.bot_token, - ) - - -@dataclass -class ChartCertificationE2ETestSingle(ChartCertificationE2ETest): - test_name: str = "" # Meaningful test name for this test, displayed in PR title - test_chart: str = "" - test_report: str = "" - chart_directory: str = "" - secrets: E2ETestSecretOneShot = E2ETestSecretOneShot() - - def __post_init__(self) -> None: - # unique string based on uuid.uuid4(), not using timestamp here because even - # in nanoseconds there are chances of collisions among first test cases of - # different processes. - self.uuid = uuid.uuid4().hex - - if self.test_report or self.test_chart: - ( - self.secrets.chart_name, - self.secrets.chart_version, - ) = self.get_chart_name_version() - self.chart_directory = f"charts/{self.secrets.vendor_type}/{self.secrets.vendor}/{self.secrets.chart_name}" - - bot_name, bot_token = self.get_bot_name_and_token() - test_repo = TEST_REPO - - # Create a new branch locally from detached HEAD - head_sha = self.repo.git.rev_parse("--short", "HEAD") - unique_branch = f"{head_sha}-{self.uuid}" - local_branches = [h.name for h in self.repo.heads] - if unique_branch not in local_branches: - self.repo.git.checkout("-b", f"{unique_branch}") - - current_branch = self.repo.active_branch.name - r = github_api("get", f"repos/{test_repo}/branches", bot_token) - branches = json.loads(r.text) - branch_names = [branch["name"] for branch in branches] - if current_branch not in branch_names: - logging.info( - f"{test_repo}:{current_branch} does not exists, creating with local branch" - ) - self.repo.git.push( - f"https://x-access-token:{bot_token}@github.com/{test_repo}", - f"HEAD:refs/heads/{current_branch}", - "-f", - ) - - pretty_test_name = self.test_name.strip().lower().replace(" ", "-") - base_branch = ( - f"{self.uuid}-{pretty_test_name}-{current_branch}" - if pretty_test_name - else f"{self.uuid}-test-{current_branch}" - ) - pr_branch = base_branch + "-pr-branch" - - self.secrets.owners_file_content = self.owners_file_content - self.secrets.test_chart = self.test_chart - self.secrets.test_report = self.test_report - self.secrets.test_repo = test_repo - self.secrets.bot_name = bot_name - self.secrets.bot_token = bot_token - self.secrets.base_branch = base_branch - self.secrets.pr_branch = pr_branch - self.secrets.index_file = "index.yaml" - self.secrets.provider_delivery = False - - def cleanup(self): - # Cleanup releases and release tags - self.cleanup_release() - # Teardown step to cleanup branches - if self.temp_dir is not None: - self.temp_dir.cleanup() - self.repo.git.worktree("prune") - - head_sha = self.repo.git.rev_parse("--short", "HEAD") - current_branch = f"{head_sha}-{self.uuid}" - logging.info(f"Delete remote '{current_branch}' branch") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/heads/{current_branch}", - self.secrets.bot_token, - ) - - logging.info(f"Delete '{self.secrets.test_repo}:{self.secrets.base_branch}'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/heads/{self.secrets.base_branch}", - self.secrets.bot_token, - ) - - logging.info( - f"Delete '{self.secrets.test_repo}:{self.secrets.base_branch}-gh-pages'" - ) - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/heads/{self.secrets.base_branch}-gh-pages", - self.secrets.bot_token, - ) - - logging.info(f"Delete '{self.secrets.test_repo}:{self.secrets.pr_branch}'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/heads/{self.secrets.pr_branch}", - self.secrets.bot_token, - ) - - logging.info(f"Delete local '{self.secrets.base_branch}'") - try: - self.repo.git.branch("-D", self.secrets.base_branch) - except git.exc.GitCommandError: - logging.info(f"Local '{self.secrets.base_branch}' does not exist") - - logging.info(f"Delete local '{current_branch}'") - try: - self.repo.git.branch("-D", current_branch) - except git.exc.GitCommandError: - logging.info(f"Local '{current_branch}' does not exist") - - def update_test_chart(self, test_chart): - if test_chart != self.test_chart: - # reinitialize the settings according with new chart - self.test_chart = test_chart - self.__post_init__() - - def update_test_report(self, test_report): - if test_report != self.test_report: - # reinitialize the settings according with new report - self.test_report = test_report - self.__post_init__() - - def get_unique_vendor(self, vendor): - """Set unique vendor name. - Note that release tag is generated with this vendor name. - """ - # unique string based on uuid.uuid4() - suffix = self.uuid - if "PR_NUMBER" in os.environ: - pr_num = os.environ["PR_NUMBER"] - suffix = f"{suffix}-{pr_num}" - return f"{vendor}-{suffix}" - - def get_chart_name_version(self): - if not self.test_report and not self.test_chart: - pytest.fail("Provide at least one of test report or test chart.") - if self.test_report: - chart_name, chart_version = get_name_and_version_from_report( - self.test_report - ) - else: - chart_name, chart_version = get_name_and_version_from_chart_tar( - self.test_chart - ) - return chart_name, chart_version - - def set_vendor(self, vendor, vendor_type): - # use unique vendor id to avoid collision between tests - self.secrets.vendor = self.get_unique_vendor(vendor) - self.secrets.vendor_type = vendor_type - base_branch_without_uuid = "-".join(self.secrets.base_branch.split("-")[:-1]) - vendor_without_suffix = self.secrets.vendor.split("-")[0] - self.secrets.base_branch = f"{base_branch_without_uuid}-{self.secrets.vendor_type}-{vendor_without_suffix}-{self.secrets.chart_name}-{self.secrets.chart_version}" - self.secrets.pr_branch = f"{self.secrets.base_branch}-pr-branch" - self.chart_directory = f"charts/{self.secrets.vendor_type}/{self.secrets.vendor}/{self.secrets.chart_name}" - - def setup_git_context(self): - super().setup_git_context(self.repo) - - def setup_gh_pages_branch(self): - self.create_test_gh_pages_branch( - self.secrets.test_repo, self.secrets.base_branch, self.secrets.bot_token - ) - - def setup_temp_dir(self): - self.temp_dir = TemporaryDirectory(prefix="tci-") - with SetDirectory(Path(self.temp_dir.name)): - # Make PR's from a temporary directory - logging.info(f"Worktree directory: {self.temp_dir.name}") - self.repo.git.worktree("add", "--detach", self.temp_dir.name, "HEAD") - self.temp_repo = git.Repo(self.temp_dir.name) - - self.set_git_username_email( - self.temp_repo, self.secrets.bot_name, GITHUB_ACTIONS_BOT_EMAIL - ) - self.temp_repo.git.checkout("-b", self.secrets.base_branch) - pathlib.Path(f"{self.chart_directory}/{self.secrets.chart_version}").mkdir( - parents=True, exist_ok=True - ) - - self.remove_chart( - self.chart_directory, - self.secrets.chart_version, - self.secrets.test_repo, - self.secrets.base_branch, - self.secrets.bot_token, - ) - self.remove_owners_file( - self.chart_directory, - self.secrets.test_repo, - self.secrets.base_branch, - self.secrets.bot_token, - ) - - def update_chart_version_in_chart_yaml(self, new_version): - with SetDirectory(Path(self.temp_dir.name)): - path = f"{self.chart_directory}/{self.secrets.chart_version}/src/Chart.yaml" - with open(path, "r") as fd: - try: - chart = yaml.safe_load(fd) - except yaml.YAMLError as err: - pytest.fail(f"error parsing '{path}': {err}") - current_version = chart["version"] - - if current_version != new_version: - chart["version"] = new_version - try: - with open(path, "w") as fd: - fd.write(yaml.dump(chart)) - except Exception as e: - pytest.fail("Failed to update version in yaml file") - - def remove_readme_file(self): - with SetDirectory(Path(self.temp_dir.name)): - path = f"{self.chart_directory}/{self.secrets.chart_version}/src/README.md" - try: - os.remove(path) - except Exception as e: - pytest.fail(f"Failed to remove readme file : {e}") - - def process_owners_file(self): - super().create_and_push_owners_file( - self.chart_directory, - self.secrets.base_branch, - self.secrets.vendor, - self.secrets.vendor_type, - self.secrets.chart_name, - self.secrets.provider_delivery, - ) - - def process_chart(self, is_tarball: bool): - with SetDirectory(Path(self.temp_dir.name)): - if is_tarball: - # Copy the chart tar into temporary directory for PR submission - chart_tar = self.secrets.test_chart.split("/")[-1] - shutil.copyfile( - f"{self.old_cwd}/{self.secrets.test_chart}", - f"{self.chart_directory}/{self.secrets.chart_version}/{chart_tar}", - ) - else: - # Unzip files into temporary directory for PR submission - extract_chart_tgz( - self.secrets.test_chart, - f"{self.chart_directory}/{self.secrets.chart_version}", - self.secrets, - logging, - ) - - def process_report( - self, - update_chart_sha=False, - update_url=False, - url=None, - update_versions=False, - supported_versions=None, - tested_version=None, - kube_version=None, - update_provider_delivery=False, - provider_delivery=False, - missing_check=None, - unset_package_digest=False, - ): - with SetDirectory(Path(self.temp_dir.name)): - # Copy report to temporary location and push to test_repo:pr_branch - logging.info( - f"Push report to '{self.secrets.test_repo}:{self.secrets.pr_branch}'" - ) - tmpl = open(self.secrets.test_report).read() - values = { - "repository": self.secrets.test_repo, - "branch": self.secrets.base_branch, - } - content = Template(tmpl).substitute(values) - - report_path = ( - f"{self.chart_directory}/{self.secrets.chart_version}/" - + self.secrets.test_report.split("/")[-1] - ) - - try: - report = yaml.safe_load(content) - except yaml.YAMLError as err: - pytest.fail(f"error parsing '{report_path}': {err}") - - if self.secrets.vendor_type != "partners": - report["metadata"]["tool"]["profile"][ - "VendorType" - ] = self.secrets.vendor_type - logging.info( - f'VendorType set to {report["metadata"]["tool"]["profile"]["VendorType"]} in report.yaml' - ) - - if ( - update_chart_sha - or update_url - or update_versions - or update_provider_delivery - or unset_package_digest - ): - # For updating the report.yaml, for chart sha mismatch scenario - if update_chart_sha: - new_sha_value = "sha256:5b85ae00b9ca2e61b2d70a59f98fd72136453b1a185676b29d4eb862981c1xyz" - logging.info( - f"Current SHA Value in report: {report['metadata']['tool']['digests']['chart']}" - ) - report["metadata"]["tool"]["digests"]["chart"] = new_sha_value - logging.info(f"Updated sha value in report: {new_sha_value}") - - # For updating the report.yaml, for invalid_url sceanrio - if update_url: - logging.info( - f"Current chart-uri in report: {report['metadata']['tool']['chart-uri']}" - ) - report["metadata"]["tool"]["chart-uri"] = url - logging.info(f"Updated chart-uri value in report: {url}") - - if update_versions: - report["metadata"]["tool"][ - "testedOpenShiftVersion" - ] = tested_version - report["metadata"]["tool"][ - "supportedOpenShiftVersions" - ] = supported_versions - report["metadata"]["chart"]["kubeversion"] = kube_version - logging.info( - f"Updated testedOpenShiftVersion value in report: {tested_version}" - ) - logging.info( - f"Updated supportedOpenShiftVersions value in report: {supported_versions}" - ) - logging.info(f"Updated kubeversion value in report: {kube_version}") - - if update_provider_delivery: - report["metadata"]["tool"][ - "providerControlledDelivery" - ] = provider_delivery - - if unset_package_digest: - del report["metadata"]["tool"]["digests"]["package"] - - with open(report_path, "w") as fd: - try: - fd.write(yaml.dump(report)) - logging.info("Report updated with new values") - except Exception as e: - pytest.fail("Failed to update report yaml with new values") - - # For removing the check for missing check scenario - if missing_check: - logging.info(f"Updating report with {missing_check}") - with open(report_path, "r+") as fd: - report_content = yaml.safe_load(fd) - results = report_content["results"] - new_results = filter(lambda x: x["check"] != missing_check, results) - report_content["results"] = list(new_results) - fd.seek(0) - yaml.dump(report_content, fd) - fd.truncate() - - self.temp_repo.git.add(report_path) - self.temp_repo.git.commit( - "-m", - f"Add {self.secrets.vendor} {self.secrets.chart_name} {self.secrets.chart_version} report", - ) - self.temp_repo.git.push( - f"https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}", - f"HEAD:refs/heads/{self.secrets.pr_branch}", - "-f", - ) - - def add_non_chart_related_file(self): - with SetDirectory(Path(self.temp_dir.name)): - path = f"{self.chart_directory}/Notes.txt" - with open(path, "w") as fd: - fd.write("This is a test file") - - def push_chart(self, is_tarball: bool, add_non_chart_file=False): - # Push chart to test_repo:pr_branch - if is_tarball: - chart_tar = self.secrets.test_chart.split("/")[-1] - self.temp_repo.git.add( - f"{self.chart_directory}/{self.secrets.chart_version}/{chart_tar}" - ) - else: - if add_non_chart_file: - self.temp_repo.git.add(f"{self.chart_directory}/") - else: - self.temp_repo.git.add( - f"{self.chart_directory}/{self.secrets.chart_version}/src" - ) - self.temp_repo.git.commit( - "-m", - f"Add {self.secrets.vendor} {self.secrets.chart_name} {self.secrets.chart_version} chart", - ) - - self.temp_repo.git.push( - f"https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}", - f"HEAD:refs/heads/{self.secrets.pr_branch}", - "-f", - ) - - def send_pull_request(self): - self.secrets.pr_number = super().send_pull_request( - self.secrets.test_repo, - self.secrets.base_branch, - self.secrets.pr_branch, - self.secrets.bot_token, - ) - logging.info(f"[INFO] PR number: {self.secrets.pr_number}") - - # expect_result: a string representation of expected result, e.g. 'success' - def check_workflow_conclusion(self, expect_result: str): - # Check workflow conclusion - super().check_workflow_conclusion(None, expect_result, pytest.fail) - - # expect_merged: boolean representing whether the PR should be merged - def check_pull_request_result(self, expect_merged: bool): - super().check_pull_request_result( - self.secrets.pr_number, expect_merged, pytest.fail - ) - - # expect_merged: boolean representing whether the PR should be merged - def check_pull_request_labels(self): - super().check_pull_request_labels(self.secrets.pr_number, pytest.fail) - - def check_pull_request_comments(self, expect_message: str): - r = github_api( - "get", - f"repos/{self.secrets.test_repo}/issues/{self.secrets.pr_number}/comments", - self.secrets.bot_token, - ) - logging.info(f"STATUS_CODE: {r.status_code}") - - response = json.loads(r.text) - complete_comment = response[0]["body"] - - if re.search(expect_message, complete_comment): - logging.info("Found the expected comment in the PR") - else: - pytest.fail( - f"Was expecting '{expect_message}' in the comment {complete_comment}" - ) - - def check_index_yaml(self, check_provider_type=False): - super().check_index_yaml( - self.secrets.base_branch, - self.secrets.vendor, - self.secrets.chart_name, - self.secrets.chart_version, - self.secrets.index_file, - check_provider_type, - pytest.fail, - ) - - def check_release_result(self): - chart_tgz = self.secrets.test_chart.split("/")[-1] - super().check_release_result( - self.secrets.vendor, - self.secrets.chart_name, - self.secrets.chart_version, - chart_tgz, - pytest.fail, - ) - - def cleanup_release(self): - expected_tag = f"{self.secrets.vendor}-{self.secrets.chart_name}-{self.secrets.chart_version}" - super().cleanup_release(expected_tag) - - -@dataclass -class ChartCertificationE2ETestMultiple(ChartCertificationE2ETest): - secrets: E2ETestSecretRecursive = E2ETestSecretRecursive() - - def __post_init__(self) -> None: - bot_name, bot_token = self.get_bot_name_and_token() - dry_run = self.get_dry_run() - notify_id = self.get_notify_id() - software_name, software_version = self.get_software_name_version() - vendor_type = self.get_vendor_type() - - test_repo = TEST_REPO - base_branches = [] - pr_branches = [] - - pr_base_branch = self.repo.active_branch.name - r = github_api("get", f"repos/{test_repo}/branches", bot_token) - branches = json.loads(r.text) - branch_names = [branch["name"] for branch in branches] - if pr_base_branch not in branch_names: - logging.info( - f"{test_repo}:{pr_base_branch} does not exists, creating with local branch" - ) - self.repo.git.push( - f"https://x-access-token:{bot_token}@github.com/{test_repo}", - f"HEAD:refs/heads/{pr_base_branch}", - "-f", - ) - - self.secrets = E2ETestSecretRecursive() - self.secrets.software_name = software_name - self.secrets.software_version = software_version - self.secrets.test_repo = test_repo - self.secrets.bot_name = bot_name - self.secrets.bot_token = bot_token - self.secrets.vendor_type = vendor_type - self.secrets.pr_base_branch = pr_base_branch - self.secrets.base_branches = base_branches - self.secrets.pr_branches = pr_branches - self.secrets.dry_run = dry_run - self.secrets.notify_id = notify_id - self.secrets.owners_file_content = self.owners_file_content - self.secrets.release_tags = list() - - def cleanup(self): - # Teardown step to cleanup releases and branches - for release_tag in self.secrets.release_tags: - self.cleanup_release(release_tag) - - self.repo.git.worktree("prune") - for base_branch in self.secrets.base_branches: - logging.info(f"Delete '{self.secrets.test_repo}:{base_branch}'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/heads/{base_branch}", - self.secrets.bot_token, - ) - - logging.info(f"Delete '{self.secrets.test_repo}:{base_branch}-gh-pages'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/heads/{base_branch}-gh-pages", - self.secrets.bot_token, - ) - - logging.info(f"Delete local '{base_branch}'") - try: - self.repo.git.branch("-D", base_branch) - except git.exc.GitCommandError: - logging.info(f"Local '{base_branch}' does not exist") - - for pr_branch in self.secrets.pr_branches: - logging.info(f"Delete '{self.secrets.test_repo}:{pr_branch}'") - github_api( - "delete", - f"repos/{self.secrets.test_repo}/git/refs/heads/{pr_branch}", - self.secrets.bot_token, - ) - - try: - logging.info("Delete local 'tmp' branch") - self.temp_repo.git.branch("-D", "tmp") - except git.exc.GitCommandError: - logging.info("Local 'tmp' branch does not exist") - - def get_dry_run(self): - # Accepts 'true' or 'false', depending on whether we want to notify - # Don't notify on dry runs, default to True - dry_run = False if os.environ.get("DRY_RUN") == "false" else True - # Don't notify if not triggerd on PROD_REPO and PROD_BRANCH - if not dry_run: - triggered_branch = os.environ.get("GITHUB_REF").split("/")[-1] - triggered_repo = os.environ.get("GITHUB_REPOSITORY") - if triggered_repo != PROD_REPO or triggered_branch != PROD_BRANCH: - dry_run = True - return dry_run - - def get_notify_id(self): - # Accepts comma separated Github IDs or empty strings to override people to tag in notifications - notify_id = os.environ.get("NOTIFY_ID") - if notify_id: - notify_id = [vt.strip() for vt in notify_id.split(",")] - else: - notify_id = ["dperaza", "mmulholla"] - return notify_id - - def get_software_name_version(self): - software_name = os.environ.get("SOFTWARE_NAME") - if not software_name: - raise Exception("SOFTWARE_NAME environment variable not defined") - - software_version = os.environ.get("SOFTWARE_VERSION").strip('"') - if not software_version: - raise Exception("SOFTWARE_VERSION environment variable not defined") - elif software_version.startswith("sha256"): - software_version = software_version[-8:] - - return software_name, software_version - - def get_vendor_type(self): - vendor_type = os.environ.get("VENDOR_TYPE") - if not vendor_type: - logging.info( - "VENDOR_TYPE environment variable not defined, default to `all`" - ) - vendor_type = "all" - return vendor_type - - def setup_temp_dir(self): - self.temp_dir = TemporaryDirectory(prefix="tci-") - with SetDirectory(Path(self.temp_dir.name)): - # Make PR's from a temporary directory - logging.info(f"Worktree directory: {self.temp_dir.name}") - self.repo.git.worktree("add", "--detach", self.temp_dir.name, "HEAD") - self.temp_repo = git.Repo(self.temp_dir.name) - - # Run submission flow test with charts in PROD_REPO:PROD_BRANCH - self.set_git_username_email( - self.temp_repo, self.secrets.bot_name, GITHUB_ACTIONS_BOT_EMAIL - ) - self.temp_repo.git.checkout(PROD_BRANCH, "charts") - self.temp_repo.git.restore("--staged", "charts") - self.secrets.submitted_charts = get_all_charts( - "charts", self.secrets.vendor_type - ) - logging.info( - f"Found charts for {self.secrets.vendor_type}: {self.secrets.submitted_charts}" - ) - self.temp_repo.git.checkout("-b", "tmp") - - def get_owner_ids(self, chart_directory, owners_table): - with open(f"{chart_directory}/OWNERS", "r") as fd: - try: - owners = yaml.safe_load(fd) - # Pick owner ids for notification - owners_table[chart_directory] = [ - owner.get("githubUsername", "") for owner in owners["users"] - ] - except yaml.YAMLError as err: - logging.warning(f"Error parsing OWNERS of {chart_directory}: {err}") - - def push_chart( - self, - chart_directory, - chart_name, - chart_version, - vendor_name, - vendor_type, - pr_branch, - ): - # Push chart files to test_repo:pr_branch - self.temp_repo.git.add(f"{chart_directory}/{chart_version}") - self.temp_repo.git.commit( - "-m", - f"Add {vendor_type} {vendor_name} {chart_name} {chart_version} chart files", - ) - self.temp_repo.git.push( - f"https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}", - f"HEAD:refs/heads/{pr_branch}", - "-f", - ) - - def report_failure( - self, chart, chart_owners, failure_type, pr_html_url=None, run_html_url=None - ): - os.environ["GITHUB_REPO"] = PROD_REPO.split("/")[1] - os.environ["GITHUB_AUTH_TOKEN"] = self.secrets.bot_token - if not self.secrets.dry_run: - os.environ["GITHUB_REPO"] = PROD_REPO.split("/")[1] - os.environ["GITHUB_AUTH_TOKEN"] = self.secrets.bot_token - os.environ["GITHUB_ORGANIZATION"] = PROD_REPO.split("/")[0] - logging.info( - f"Send notification to '{self.secrets.notify_id}' about verification result of '{chart}'" - ) - create_verification_issue( - chart, - chart_owners, - failure_type, - self.secrets.notify_id, - pr_html_url, - run_html_url, - self.secrets.software_name, - self.secrets.software_version, - self.secrets.bot_token, - self.secrets.dry_run, - ) - else: - os.environ["GITHUB_ORGANIZATION"] = PROD_REPO.split("/")[0] - os.environ["GITHUB_REPO"] = "sandbox" - os.environ["GITHUB_AUTH_TOKEN"] = self.secrets.bot_token - logging.info( - f"Send notification to '{self.secrets.notify_id}' about dry run verification result of '{chart}'" - ) - create_verification_issue( - chart, - chart_owners, - failure_type, - self.secrets.notify_id, - pr_html_url, - run_html_url, - self.secrets.software_name, - self.secrets.software_version, - self.secrets.bot_token, - self.secrets.dry_run, - ) - logging.info( - f"Dry Run - send sandbox notification to '{chart_owners}' about verification result of '{chart}'" - ) - - def check_single_chart_result( - self, - vendor_type, - vendor_name, - chart_name, - chart_version, - pr_number, - owners_table, - ): - base_branch = f"{self.secrets.software_name}-{self.secrets.software_version}-{self.secrets.pr_base_branch}-{vendor_type}-{vendor_name}-{chart_name}-{chart_version}" - - # Check workflow conclusion - chart = f"{vendor_name} {chart_name} {chart_version}" - run_id, conclusion = super().check_workflow_conclusion( - pr_number, "success", logging.warning - ) - - if conclusion and run_id: - if conclusion != "success": - # Send notification to owner through GitHub issues - r = github_api( - "get", - f"repos/{self.secrets.test_repo}/actions/runs/{run_id}", - self.secrets.bot_token, - ) - run = r.json() - run_html_url = run["html_url"] - - pr = get_pr(self.secrets, pr_number) - pr_html_url = pr["html_url"] - chart_directory = f"charts/{vendor_type}/{vendor_name}/{chart_name}" - chart_owners = owners_table[chart_directory] - - self.report_failure( - chart, chart_owners, CHECKS_FAILED, pr_html_url, run_html_url - ) - - logging.warning( - f"PR{pr_number} workflow failed: {vendor_name}, {chart_name}, {chart_version}" - ) - return - else: - logging.info( - f"PR{pr_number} workflow passed: {vendor_name}, {chart_name}, {chart_version}" - ) - else: - logging.warning( - f"PR{pr_number} workflow did not complete: {vendor_name}, {chart_name}, {chart_version}" - ) - return - - # Check PRs are merged - if not super().check_pull_request_result(pr_number, True, logging.warning): - logging.warning( - f"PR{pr_number} pull request was not merged: {vendor_name}, {chart_name}, {chart_version}" - ) - return - logging.info( - f"PR{pr_number} pull request was merged: {vendor_name}, {chart_name}, {chart_version}" - ) - - # Check index.yaml is updated - if not super().check_index_yaml( - base_branch, - vendor_name, - chart_name, - chart_version, - check_provider_type=False, - logger=logging.warning, - ): - logging.warning( - f"PR{pr_number} - Chart was not found in Index file: {vendor_name}, {chart_name}, {chart_version}" - ) - logging.info( - f"PR{pr_number} - Chart was found in Index file: {vendor_name}, {chart_name}, {chart_version}" - ) - - # Check release is published - chart_tgz = f"{chart_name}-{chart_version}.tgz" - if not super().check_release_result( - vendor_name, chart_name, chart_version, chart_tgz, logging.warning - ): - logging.warning( - f"PR{pr_number} - Release was not created: {vendor_name}, {chart_name}, {chart_version}" - ) - logging.info( - f"PR{pr_number} - Release was created: {vendor_name}, {chart_name}, {chart_version}" - ) - - def process_single_chart( - self, - vendor_type, - vendor_name, - chart_name, - chart_version, - pr_number_list, - owners_table, - ): - # Get SHA from 'pr_base_branch' branch - logging.info( - f"Process chart: {vendor_type}/{vendor_name}/{chart_name}/{chart_version}" - ) - r = github_api( - "get", - f"repos/{self.secrets.test_repo}/git/ref/heads/{self.secrets.pr_base_branch}", - self.secrets.bot_token, - ) - j = json.loads(r.text) - pr_base_branch_sha = j["object"]["sha"] - - chart_directory = f"charts/{vendor_type}/{vendor_name}/{chart_name}" - base_branch = f"{self.secrets.software_name}-{self.secrets.software_version}-{self.secrets.pr_base_branch}-{vendor_type}-{vendor_name}-{chart_name}-{chart_version}" - base_branch = base_branch.replace(":", "-") - pr_branch = f"{base_branch}-pr-branch" - - self.secrets.base_branches.append(base_branch) - self.secrets.pr_branches.append(pr_branch) - self.temp_repo.git.checkout("tmp") - self.temp_repo.git.checkout("-b", base_branch) - - # Create test gh-pages branch for checking index.yaml - self.create_test_gh_pages_branch( - self.secrets.test_repo, base_branch, self.secrets.bot_token - ) - - # Create a new base branch for testing current chart - logging.info(f"Create {self.secrets.test_repo}:{base_branch} for testing") - r = github_api( - "get", f"repos/{self.secrets.test_repo}/branches", self.secrets.bot_token - ) - branches = json.loads(r.text) - branch_names = [branch["name"] for branch in branches] - if base_branch in branch_names: - logging.warning(f"{self.secrets.test_repo}:{base_branch} already exists") - return - data = {"ref": f"refs/heads/{base_branch}", "sha": pr_base_branch_sha} - r = github_api( - "post", - f"repos/{self.secrets.test_repo}/git/refs", - self.secrets.bot_token, - json=data, - ) - - # Remove chart and owners file from git - self.remove_chart( - chart_directory, - chart_version, - self.secrets.test_repo, - base_branch, - self.secrets.bot_token, - ) - self.remove_owners_file( - chart_directory, self.secrets.test_repo, base_branch, self.secrets.bot_token - ) - - # Get owners id for notifications - self.get_owner_ids(chart_directory, owners_table) - - # Create and push test owners file - super().create_and_push_owners_file( - chart_directory, base_branch, vendor_name, vendor_type, chart_name - ) - - # Push test chart to pr_branch - self.push_chart( - chart_directory, - chart_name, - chart_version, - vendor_name, - vendor_type, - pr_branch, - ) - - # Create PR from pr_branch to base_branch - logging.info("sleep for 5 seconds to avoid secondary api limit") - time.sleep(5) - pr_number = super().send_pull_request( - self.secrets.test_repo, base_branch, pr_branch, self.secrets.bot_token - ) - pr_number_list.append( - (vendor_type, vendor_name, chart_name, chart_version, pr_number) - ) - logging.info( - f"PR{pr_number} created in {self.secrets.test_repo} into {base_branch} from {pr_branch}" - ) - - # Record expected release tags - self.secrets.release_tags.append(f"{vendor_name}-{chart_name}-{chart_version}") - - def process_all_charts(self): - self.setup_git_context(self.repo) - self.setup_temp_dir() - - owners_table = dict() - pr_number_list = list() - - skip_charts = list() - - logging.info( - f"Running tests for : {self.secrets.software_name} {self.secrets.software_version} :" - ) - # First look for charts in index.yaml to see if kubeVersion is good: - if self.secrets.software_name == "OpenShift": - logging.info("check index file for invalid kubeVersions") - failed_charts = check_index_entries(self.secrets.software_version) - if failed_charts: - for chart in failed_charts: - providerDir = chart["providerType"].replace("partner", "partners") - chart_directory = ( - f'charts/{providerDir}/{chart["provider"]}/{chart["name"]}' - ) - self.get_owner_ids(chart_directory, owners_table) - chart_owners = owners_table[chart_directory] - chart_id = f'{chart["provider"]} {chart["name"]} {chart["version"]}' - self.report_failure( - chart_id, chart_owners, chart["message"], "", "" - ) - skip_charts.append(f'{chart["name"]}-{chart["version"]}') - - # Process test charts and send PRs from temporary directory - with SetDirectory(Path(self.temp_dir.name)): - for ( - vendor_type, - vendor_name, - chart_name, - chart_version, - ) in self.secrets.submitted_charts: - if f"{chart_name}-{chart_version}" in skip_charts: - logging.info( - f"Skip already failed chart: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}" - ) - else: - logging.info( - f"Process chart: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}" - ) - self.process_single_chart( - vendor_type, - vendor_name, - chart_name, - chart_version, - pr_number_list, - owners_table, - ) - logging.info("sleep for 5 seconds to avoid secondary api limit") - time.sleep(5) - - for ( - vendor_type, - vendor_name, - chart_name, - chart_version, - pr_number, - ) in pr_number_list: - logging.info( - f"PR{pr_number} Check result: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}" - ) - self.check_single_chart_result( - vendor_type, - vendor_name, - chart_name, - chart_version, - pr_number, - owners_table, - ) diff --git a/tests/functional/utils/github.py b/tests/functional/utils/github.py deleted file mode 100644 index f91ecc5231..0000000000 --- a/tests/functional/utils/github.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- -"""Utility class for setting up and manipulating GitHub operations.""" - -import json -import requests -from retrying import retry - -from functional.utils.setttings import * - - -@retry(stop_max_delay=30_000, wait_fixed=1000) -def get_run_id(secrets, pr_number=None): - pr = get_pr(secrets, pr_number) - r = github_api("get", f"repos/{secrets.test_repo}/actions/runs", secrets.bot_token) - runs = json.loads(r.text) - - for run in runs["workflow_runs"]: - if ( - run["head_sha"] == pr["head"]["sha"] - and run["name"] == CERTIFICATION_CI_NAME - ): - return run["id"] - else: - raise Exception("Workflow for the submitted PR did not run.") - - -@retry(stop_max_delay=60_000 * 40, wait_fixed=2000) -def get_run_result(secrets, run_id): - r = github_api( - "get", f"repos/{secrets.test_repo}/actions/runs/{run_id}", secrets.bot_token - ) - run = json.loads(r.text) - - if run["conclusion"] is None: - raise Exception(f"Workflow {run_id} is still running, PR: {secrets.pr_number} ") - - return run["conclusion"] - - -@retry(stop_max_delay=10_000, wait_fixed=1000) -def get_release_assets(secrets, release_id, required_assets): - r = github_api( - "get", - f"repos/{secrets.test_repo}/releases/{release_id}/assets", - secrets.bot_token, - ) - asset_list = json.loads(r.text) - asset_names = [asset["name"] for asset in asset_list] - missing_assets = list() - for asset in required_assets: - if asset not in asset_names: - missing_assets.append(asset) - if len(missing_assets) > 0: - raise Exception(f"Missing release asset: {missing_assets}") - - -@retry(stop_max_delay=15_000, wait_fixed=1000) -def get_release_by_tag(secrets, release_tag): - r = github_api("get", f"repos/{secrets.test_repo}/releases", secrets.bot_token) - releases = json.loads(r.text) - for release in releases: - if release["tag_name"] == release_tag: - return release - raise Exception("Release not published") - - -def get_pr(secrets, pr_number=None): - pr_number = secrets.pr_number if pr_number is None else pr_number - r = github_api( - "post", f"repos/{secrets.test_repo}/pulls/{pr_number}", secrets.bot_token - ) - pr = json.loads(r.text) - return pr - - -def github_api_get(endpoint, bot_token, headers={}): - if not headers: - headers = { - "Accept": "application/vnd.github.v3+json", - "Authorization": f"Bearer {bot_token}", - } - r = requests.get(f"{GITHUB_BASE_URL}/{endpoint}", headers=headers) - - return r - - -def github_api_delete(endpoint, bot_token, headers={}): - if not headers: - headers = { - "Accept": "application/vnd.github.v3+json", - "Authorization": f"Bearer {bot_token}", - } - r = requests.delete(f"{GITHUB_BASE_URL}/{endpoint}", headers=headers) - - return r - - -def github_api_post(endpoint, bot_token, headers={}, json={}): - if not headers: - headers = { - "Accept": "application/vnd.github.v3+json", - "Authorization": f"Bearer {bot_token}", - } - r = requests.post(f"{GITHUB_BASE_URL}/{endpoint}", headers=headers, json=json) - - return r - - -def github_api(method, endpoint, bot_token, headers={}, data={}, json={}): - if method == "get": - return github_api_get(endpoint, bot_token, headers=headers) - elif method == "post": - return github_api_post(endpoint, bot_token, headers=headers, json=json) - elif method == "delete": - return github_api_delete(endpoint, bot_token, headers=headers) - else: - raise ValueError("Github API method not implemented in helper function") diff --git a/tests/functional/utils/index.py b/tests/functional/utils/index.py deleted file mode 100644 index 45ad4ab8a2..0000000000 --- a/tests/functional/utils/index.py +++ /dev/null @@ -1,49 +0,0 @@ -import logging -import semantic_version -import sys - -sys.path.append("../../../scripts/src") -from chartrepomanager import indexannotations -from indexfile import index - - -def check_index_entries(ocpVersion): - all_chart_list = index.get_latest_charts() - failed_chart_list = [] - - OCP_VERSION = semantic_version.Version.coerce(ocpVersion) - - for chart in all_chart_list: - if ( - "supportedOCP" in chart - and chart["supportedOCP"] != "N/A" - and chart["supportedOCP"] != "" - ): - if OCP_VERSION in semantic_version.NpmSpec(chart["supportedOCP"]): - logging.info( - f'PASS: Chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} includes: {OCP_VERSION}' - ) - else: - chart[ - "message" - ] = f'chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} does not include latest OCP version {OCP_VERSION}' - logging.info( - f' ERROR: Chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} does not include {OCP_VERSION}' - ) - failed_chart_list.append(chart) - elif "kubeVersion" in chart and chart["kubeVersion"] != "": - supportedOCPVersion = indexannotations.getOCPVersions(chart["kubeVersion"]) - if OCP_VERSION in semantic_version.NpmSpec(supportedOCPVersion): - logging.info( - f'PASS: Chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) includes OCP version: {OCP_VERSION}' - ) - else: - chart[ - "message" - ] = f'chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) does not include latest OCP version {OCP_VERSION}' - logging.info( - f' ERROR: Chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) does not include {OCP_VERSION}' - ) - failed_chart_list.append(chart) - - return failed_chart_list diff --git a/tests/functional/utils/notifier.py b/tests/functional/utils/notifier.py deleted file mode 100755 index 923859fd4a..0000000000 --- a/tests/functional/utils/notifier.py +++ /dev/null @@ -1,206 +0,0 @@ -# -*- coding: utf-8 -*- -"""Utility module for sending chart owners notifications.""" - -import json -import os -import sys - -import requests - -from functional.utils.setttings import * - -endpoint_data = {} - -CHECKS_FAILED = "checks failed" - - -def _set_endpoint_key(key, env_var): - if key not in endpoint_data: - if env_var in os.environ: - endpoint_data[key] = os.environ[env_var] - else: - raise Exception( - f"Environment variables {env_var} is required to connect to github" - ) - - -def _set_endpoint(): - _set_endpoint_key("access_token", "GITHUB_AUTH_TOKEN") - _set_endpoint_key("organization", "GITHUB_ORGANIZATION") - _set_endpoint_key("repo", "GITHUB_REPO") - - -def _make_gihub_request(method, uri, body=None, params={}, headers={}, verbose=False): - headers.update( - { - "Authorization": f'Bearer {endpoint_data["access_token"]}', - "Accept": "application/vnd.github.v3+json", - } - ) - - url = f'{GITHUB_BASE_URL}/repos/{endpoint_data["organization"]}/{endpoint_data["repo"]}/{uri}' - - print(f"API url: {url}") - method_map = { - "get": requests.get, - "post": requests.post, - "put": requests.put, - "delete": requests.delete, - "patch": requests.patch, - } - request_method = method_map[method] - response = request_method(url, params=params, headers=headers, json=body) - if verbose: - print(json.dumps(headers, indent=4, sort_keys=True)) - print(json.dumps(body, indent=4, sort_keys=True)) - print(json.dumps(params, indent=4, sort_keys=True)) - print(response.text) - response.raise_for_status() - try: - resp_json = response.json() - except Exception: - resp_json = None - if resp_json and verbose: - print(json.dumps(resp_json, indent=4, sort_keys=True)) - return resp_json - - -# Call this method directly if you are not creating a verification issue nor a version change issue. -def create_an_issue(title, description, assignees=[], labels=[]): - uri = "issues" - method = "post" - body = { - "title": title, - "body": description, - "assignees": assignees, - "labels": labels, - } - _make_gihub_request(method, uri, body=body, verbose=False) - - -def _verify_endpoint(access_token): - if "repo" not in endpoint_data: - raise Exception("GITHUB_REPO environment variable not defined") - - if "organization" not in endpoint_data: - raise Exception("GITHUB_ORGANIZATION environment variable not defined") - - if access_token: - endpoint_data["access_token"] = access_token - - -def create_verification_issue( - chart, - chart_owners, - failure_type, - notify_developers, - pr_url, - report_url, - software_name, - software_version, - access_token=None, - dry_run=False, -): - """Create and issue with chart-verifier findings after a version change trigger. - - chart_name -- Name of the chart that was verified. Include version for more verbose information\n - chart_owners -- Github IDs of the chart owners\n - failure_type - Indication of the type of failure - report_url -- URL or the report resulting from verification if applicable\n - kube-version -- The kubeVersion attribute of the chart if it is bade.\n - software_name -- Name of the software dependency that changed e.g, OCP and Chart Verifier\n - software_version -- The softwared dependency version used\n - access_token -- An optional github access token secret. If not passed will try to get from GITHUB_AUTH_TOKEN environment variable\ - dry-run -- Set if the test run is a dry-run. - """ - - title = f"Chart {chart}" - if dry_run: - title = f"Dry Run: Chart {chart}" - - if failure_type == CHECKS_FAILED: - title = f"{title} has failures with {software_name} version {software_version}" - report_result = "some chart checks have failed. Please review the failures and, if required, consider submitting a new chart version with the appropriate additions/corrections." - body = ( - f"FYI @{' @'.join(notify_developers)}, in PR {pr_url} we triggered the chart certification workflow against chart {chart} because the workflow " - f"now supports {software_name} version {software_version}. We have found that {report_result}. Check details in the report: " - f"{report_url}, Chart owners are: {chart_owners}" - ) - else: - title = f"{title} does not support {software_name} version {software_version}" - body = ( - f"FYI @{' @'.join(notify_developers)}, we checked the OCP versions supported by {chart} because the workflow " - f"now supports {software_name} version {software_version}. We have found that {failure_type}. Chart owners are: {chart_owners}" - ) - - _set_endpoint() - _verify_endpoint(access_token) - create_an_issue(title, body) - - -def create_version_change_issue( - chart_name, chart_owners, software_name, software_version, access_token=None -): - """Create and issue with new version of software dependencies supported by certitifcation program. - - chart_name -- Name of the chart afected. Include version for more verbose information - chart_owners -- Github IDs of the chart owners\n - software_name -- Name of the software dependency that changed e.g, OCP and Chart Verifier\n - software_version -- The softwared dependency version used\n - access_token -- An optional github access token secret. If not passed will try to get from GITHUB_AUTH_TOKEN environment variable\n - """ - - title = f"Action needed for {chart_name} after a certification dependency change" - - body = ( - f"FYI @{' @'.join(chart_owners)}, {software_name} {software_version} is now supported by the certification program. " - "Consider submiting a new chart version." - ) - - _set_endpoint() - _verify_endpoint(access_token) - create_an_issue(title, body) - - -if __name__ == "__main__": - # Collecting info interactively - print("Enter the chart name: ") - chart_name = sys.stdin.readline().strip() - print("Enter chart owners: ") - chart_owners = sys.stdin.readline().strip().split() - print("Enter the github organization: ") - organization = sys.stdin.readline().strip() - print("Enter the github repo: ") - repo = sys.stdin.readline().strip() - - # setting endpoint - print(f"Creating custom issue in https://github.com/{organization}/{repo}") - endpoint_data["organization"] = organization - endpoint_data["repo"] = repo - - print("Enter the name of software dependency that changed: ") - software_name = sys.stdin.readline().strip() - print("Enter the version of software dependency that changed: ") - software_version = sys.stdin.readline().strip() - - print("What type of issue are you creating (verification/version-change)?: ") - issue_type = sys.stdin.readline().strip() - - if issue_type == "verification": - print("Enter the report url: ") - report_url = sys.stdin.readline().strip() - print("Did the chart verification pass (yes/no)?: ") - pass_answer = sys.stdin.readline().strip() - pass_verification = pass_answer == "yes" - create_verification_issue( - chart_name, - chart_owners, - report_url, - software_name, - software_version, - pass_verification=pass_verification, - ) - else: - create_version_change_issue( - chart_name, chart_owners, software_name, software_version - ) diff --git a/tests/functional/utils/secret.py b/tests/functional/utils/secret.py deleted file mode 100644 index 91a9d49f44..0000000000 --- a/tests/functional/utils/secret.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -"""Utility class for storing test specific settings.""" - -from dataclasses import dataclass - - -@dataclass -class E2ETestSecret: - # common secrets between one-shot and recursive tests - test_repo: str = "" - bot_name: str = "" - bot_token: str = "" - pr_number: int = -1 - vendor_type: str = "" - owners_file_content: str = "" - test_chart: str = "" - test_report: str = "" - chart_name: str = "" - chart_version: str = "" - - -@dataclass -class E2ETestSecretOneShot(E2ETestSecret): - # one-shot testing - base_branch: str = "" - pr_branch: str = "" - pr_number: int = -1 - vendor: str = "" - bad_version: str = "" - provider_delivery: bool = False - index_file: str = "index.yaml" - - -@dataclass -class E2ETestSecretRecursive(E2ETestSecret): - # recursive testing - software_name: str = "" - software_version: str = "" - pr_base_branch: str = "" - base_branches: list = None - pr_branches: list = None - dry_run: bool = True - notify_id: list = None - submitted_charts: list = None - release_tags: list = None diff --git a/tests/functional/utils/set_directory.py b/tests/functional/utils/set_directory.py deleted file mode 100644 index becb220e5a..0000000000 --- a/tests/functional/utils/set_directory.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -"""Sets the cwd within the context. - -Reference: https://dev.to/teckert/changing-directory-with-a-python-context-manager-2bj8 -""" - -import os -from dataclasses import dataclass -from pathlib import Path - - -@dataclass -class SetDirectory(object): - """ - Args: - path (Path): The path to the cwd - """ - - path: Path - origin: Path = Path().absolute() - - def __enter__(self): - os.chdir(self.path) - - def __exit__(self, exc_type, exc_value, traceback): - os.chdir(self.origin) diff --git a/tests/functional/utils/setttings.py b/tests/functional/utils/setttings.py deleted file mode 100644 index 45caafc860..0000000000 --- a/tests/functional/utils/setttings.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -"""Settings and global variables for e2e tests""" - -GITHUB_BASE_URL = "https://api.github.com" -# The sandbox repository where we run all our tests on -TEST_REPO = "openshift-helm-charts/sandbox" -# The prod repository where we create notification issues -PROD_REPO = "openshift-helm-charts/charts" -# The prod branch where we store all chart files -PROD_BRANCH = "main" -# This is used to find chart certification workflow run id -CERTIFICATION_CI_NAME = "CI" -# GitHub actions bot email for git email -GITHUB_ACTIONS_BOT_EMAIL = "41898282+github-actions[bot]@users.noreply.github.com"