Extract PR validation from chart-verifier job

This PR refactors the certification pipeline by splitting the
chart-verifier job into three jobs, running sequentially:
- The new 'validate-submission' is responsible for extracting and
  validating all information related to a PR.
- The existing 'chart-verifier' job is now only responsible for running
  chart-verifier against the Chart source (if provided, or verifies the
  submitted report.yaml instead).
- The new 'manage-gh-pr' adds a comment on the PR, and merges it in case
  the pipeline was successfull.

This PR also introduces a new mechanism for passing information between
jobs. In addition to the existing GitHub outputs, a submission.json
artifact containing a json representation of the Submission object is
created in 'validate-submission'. Subsequent steps/jobs now have the
possibility to read from this file.

The main pursued goal behind this PR is to better separate concerns and
avoid mixed up logic, thus improving readability and clarity of our
pipeline and improve future maintainability and onboarding.

This PR also opens up the ability to reducde code deduplication and
redundant checks. In the interest of keeping this PR to a minimum, this
has not been fully done here. One example: get-verify-params.py
currently queries the GitHub API in order to get chart's information,
while they are already available in the submission.json artifact. This
rather trivial change is left for a future PR.

Note that the 'submission' python module is essentially replacing the
old 'checkpr' module. The only function remaining used by other modules
being 'get_file_match_compiled_patterns', it is moved to the 'reporegex'
module.

Closes #296

Signed-off-by: Matthias Goerens <[email protected]>

scripts/setup.cfg

@@ -33,7 +33,7 @@ where = src
 console_scripts =
     chart-repo-manager = chartrepomanager.chartrepomanager:main
     chart-pr-review = chartprreview.chartprreview:main
-    check-pr-content = checkprcontent.checkpr:main
+    validate-submission = submission.validate:main
     pr-artifact = pullrequest.prartifact:main
     pr-comment = pullrequest.prepare_pr_comment:main
     sa-for-chart-testing = saforcharttesting.saforcharttesting:main

scripts/src/metrics/pushowners.py

@@ -23,7 +23,7 @@ def getFileContent(changed_file):
     try:
         owner_data = owners_file.get_owner_data_from_file(changed_file)
     except owners_file.OwnersFileError as e:
-        print("Exception loading OWNERS file: {e}")
+        print(f"Exception loading OWNERS file: {e}")
         return "", "", "", "", ""
 
     users_included = owners_file.get_users_included(owner_data)

scripts/src/pullrequest/prartifact.py

@@ -1,13 +1,11 @@
 import argparse
 import os
-import pathlib
-import shutil
 import sys
 
 import requests
 
 sys.path.append("../")
-from checkprcontent import checkpr
+from reporegex import matchers
 from tools import gitutils
 
 pr_files = []
@@ -19,7 +17,7 @@
 # TODO(baijum): Move this code under chartsubmission.chart module
 def get_modified_charts(api_url):
     files = get_modified_files(api_url)
-    pattern, _, _ = checkpr.get_file_match_compiled_patterns()
+    pattern, _, _ = matchers.get_file_match_compiled_patterns()
     for file in files:
         match = pattern.match(file)
         if match:
@@ -95,24 +93,6 @@ def get_labels(api_url):
     return pr_labels
 
 
-def save_metadata(directory, vendor_label, chart, number):
-    with open(os.path.join(directory, "vendor"), "w") as fd:
-        print(f"add {directory}/vendor as {vendor_label}")
-        fd.write(vendor_label)
-
-    with open(os.path.join(directory, "chart"), "w") as fd:
-        print(f"add {directory}/chart as {chart}")
-        fd.write(chart)
-
-    with open(os.path.join(directory, "NR"), "w") as fd:
-        fd.write(number)
-
-    if os.path.exists("report.yaml"):
-        shutil.copy("report.yaml", directory)
-    else:
-        pathlib.Path(os.path.join(directory, "report.yaml")).touch()
-
-
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -148,10 +128,6 @@ def main():
         pr_files = get_modified_files(args.api_url)
         print(f"[INFO] files in pr: {pr_files}")
         gitutils.add_output("pr_files", pr_files)
-    else:
-        os.makedirs(args.directory, exist_ok=True)
-        category, organization, chart, version = get_modified_charts(args.api_url)
-        save_metadata(args.directory, organization, chart, args.number)
 
 
 if __name__ == "__main__":