feat: split the runner and tests

README.md

@@ -2,9 +2,9 @@
 [![Coverage][badge-coverage]][badge-url-coverage]
 [![Python 3.6][badge-python]](https://www.python.org)
 
-# TES Compliance Suite
+# API Compliance Suite
 
-The TES Compliance Suite determines a server's compliance with the [TES API specification][res-tes-spec]. The specification has been developed by the [Global Alliance for Genomics and Health][res-ga4gh], an international coalition, formed to enable the sharing of genomic and clinical data. It serves to provide a standardized API framework and data structure to allow for interoperability of datasets hosted at different institutions.
+The API Compliance Suite determines a server's compliance with the [TES API specification][res-tes-spec]. The specification has been developed by the [Global Alliance for Genomics and Health][res-ga4gh], an international coalition, formed to enable the sharing of genomic and clinical data. It serves to provide a standardized API framework and data structure to allow for interoperability of datasets hosted at different institutions.
 
 ## Description
 

compliance_suite/cli.py

@@ -16,6 +16,7 @@
 from compliance_suite.functions.log import logger
 from compliance_suite.job_runner import JobRunner
 from compliance_suite.report_server import ReportServer
+from compliance_suite.suite_validator import SuiteValidator
 
 
 @click.group()
@@ -43,18 +44,45 @@ def validate_regex(ctx: Any, param: Any, value: List[str]):
         raise click.BadParameter("Only letters (a-z, A-Z), digits (0-9) and underscores (_) are allowed.")
 
 
-@main.command(help='Run TES compliance tests against the servers')
+# def update_path(ctx: Any, param: Any, value: List[str]):
+#     """Update the test path wrt GitHub workspace
+#
+#     Args:
+#         ctx: The current click context
+#         param: The click parameter
+#         value: The value to validate
+#
+#     Returns:
+#         The updated value with correct file path inside GitHub workspace
+#     """
+#
+#     modified_value = ["tmp/testdir/" + path for path in value]
+#     return modified_value
+
+
+@main.command(help='Validate compliance tests')
+def validate() -> None:
+    """Validate the test suite"""
+
+    logger.info("Initiate test suite validation")
+    SuiteValidator.validate()
+    logger.info("Test suite validation finished")
+
+
+@main.command(help='Run API compliance tests against the servers')
 @click.option('--server', '-s', required=True, type=str, prompt="Enter server",
               help='server URL on which the compliance tests are run. Format - https://<url>/')
 @click.option('--version', '-v', required=True, type=str, prompt="Enter version",
-              help='TES version. Example - "1.0.0"')
+              help='API version. Example - "1.0.0"')
 @click.option('--include-tags', '-i', 'include_tags', multiple=True,
               help='run tests for provided tags', callback=validate_regex)
 @click.option('--exclude-tags', '-e', 'exclude_tags', multiple=True,
               help='skip tests for provided tags', callback=validate_regex)
 @click.option('--test-path', '-tp', 'test_path', multiple=True,
               help='the absolute or relative path of the tests to be run', default=["tests"])
-@click.option('--output_path', '-o', help='path to output the JSON report')
+# @click.option('--test-path', '-tp', 'test_path', multiple=True,
+#               help='the absolute or relative path of the tests to be run', default=["tests"], callback=update_path)
+@click.option('--output_path', '-o', help='the absolute directory path to store the JSON report')
 @click.option('--serve', default=False, is_flag=True, help='spin up a server')
 @click.option('--port', default=15800, help='port at which the compliance report is served')
 @click.option('--uptime', '-u', default=3600, help='time that server will remain up in seconds')
@@ -72,12 +100,12 @@ def report(server: str,
 
     Args:
         server (str): The server URL on which the compliance suite will be run. Format - https://<url>/
-        version (str): The compliance suite will be run against this TES version. Example - "1.0.0"
+        version (str): The compliance suite will be run against this API version. Example - "1.0.0"
         include_tags (List[str]): The list of the tags for which the compliance suite will be run.
         exclude_tags (List[str]): The list of the tags for which the compliance suite will not be run.
         test_path: The list of absolute or relative paths from the project root of the test file/directory.
             Default - ["tests"]
-        output_path (str): The output path to store the JSON compliance report
+        output_path (str): The absolute directory path to store the JSON compliance report
         serve (bool): If true, runs a local server and displays the JSON report in webview
         port (int): Set the local server port. Default - 16800
         uptime (int): The local server duration in seconds. Default - 3600 seconds
@@ -108,11 +136,11 @@ def report(server: str,
             output.write(json_report)
 
     # Writing a report copy to web dir for local server
-    with open(os.path.join(os.getcwd(), "compliance_suite", "web", "web_report.json"), "w+") as output:
+    with open(os.path.join(os.getcwd(), "web_report.json"), "w+") as output:
         output.write(json_report)
 
     if serve is True:
-        report_server = ReportServer(os.path.join(os.getcwd(), "compliance_suite", "web"))
+        report_server = ReportServer(os.getcwd())
         report_server.serve_thread(port, uptime)
 
 

compliance_suite/constants/constants.py

@@ -11,29 +11,6 @@
     'SUMMARY': 45
 }
 
-# API Constants
-# 1. Basic & Full views have same required fields. Hence, validating Basic views against Full view Model.
-
-ENDPOINT_TO_MODEL = {
-    'service_info': 'TesServiceInfo',
-    'list_tasks_MINIMAL': 'TesListTasksResponseMinimal',
-    'list_tasks_BASIC': 'TesListTasksResponse',
-    'list_tasks_FULL': 'TesListTasksResponse',
-    'get_task_MINIMAL': 'TesTaskMinimal',
-    'get_task_BASIC': 'TesTask',
-    'get_task_FULL': 'TesTask',
-    'create_task': 'TesCreateTaskResponse',
-    'create_task_request_body': 'TesTask',
-    'cancel_task': 'TesCancelTaskResponse'
-}
-
-REQUEST_HEADERS = {
-    'TES': {
-        'Accept': 'application/json',
-        'Content-Type': 'application/json'
-    }
-}
-
 # String Constants
 
 PATTERN_HASH_CENTERED = "{:#^120}"

compliance_suite/exceptions/compliance_exception.py

@@ -29,9 +29,8 @@ def __init__(self, name: str, message: str, details: Any, _type: str):
 
 
 class TestFailureException(BasicException):
-    """ When a test fails due to incomplete/wrong implementation of a TES server. The exception
-    highlights the possible changes required by the TES server to follow the standard
-    TES API Specs"""
+    """ When a test fails due to incomplete/wrong implementation of an API server. The exception
+    highlights the possible changes required by the API server to follow the standard API Specs"""
 
     def __init__(self, name: str, message: str, details: Any):
         """Initialize Test Failure Exception object

compliance_suite/functions/client.py

@@ -13,7 +13,6 @@
 import requests
 from requests.models import Response
 
-from compliance_suite.constants.constants import REQUEST_HEADERS
 from compliance_suite.exceptions.compliance_exception import (
     TestFailureException,
     TestRunnerException
@@ -28,10 +27,17 @@ def __init__(self):
         """ Initialize the Client object"""
 
         self.check_cancel = False   # Checks if the Cancel status is to be validated or not
+        self.request_headers: Dict = {}
+
+    def set_request_headers(self, request_headers) -> None:
+        """ Set the request headers
+            Args:
+                request_headers: The request headers extracted from Tests repo API config
+        """
+        self.request_headers = request_headers
 
     def send_request(
             self,
-            service: str,
             server: str,
             version: str,
             endpoint: str,
@@ -43,7 +49,6 @@ def send_request(
         """ Sends the REST request to provided server
 
         Args:
-            service (str): The GA4GH service name (eg. TES)
             server (str): The server URL to send the request
             version (str): The version of the deployed server
             endpoint (str): The endpoint of the given server
@@ -61,15 +66,14 @@ def send_request(
 
         version = "v" + version.split(".")[0]  # Convert SemVer into Major API version
         base_url: str = str(server) + version + endpoint
-        request_headers: dict = REQUEST_HEADERS[service]
         response = None
         logger.info(f"Sending {operation} request to {base_url}. Query Parameters - {query_params}")
         try:
             if operation == "GET":
-                response = requests.get(base_url, headers=request_headers, params=query_params)
+                response = requests.get(base_url, headers=self.request_headers, params=query_params)
             elif operation == "POST":
                 request_body = json.loads(request_body)
-                response = requests.post(base_url, headers=request_headers, json=request_body)
+                response = requests.post(base_url, headers=self.request_headers, json=request_body)
             return response
         except OSError as err:
             raise TestRunnerException(name="OS Error",
@@ -92,7 +96,7 @@ def check_poll(
         if response.status_code != 200:
             logger.info("Unexpected response from Polling request. Retrying...")
             return False
-
+        # TODO
         response_json: Any = response.json()
         valid_states = ["CANCELED", "CANCELING"] if self.check_cancel else ["COMPLETE", "EXECUTOR_ERROR",
                                                                             "SYSTEM_ERROR", "PREEMPTED"]
@@ -105,7 +109,6 @@ def check_poll(
 
     def poll_request(
             self,
-            service: str,
             server: str,
             version: str,
             endpoint: str,
@@ -119,7 +122,6 @@ def poll_request(
         """ This function polls a request to specified server with given interval and timeout
 
         Args:
-            service (str): The GA4GH service name (eg. TES)
             server (str): The server URL to send the request
             version (str): The version of the deployed server
             endpoint (str): The endpoint of the given server
@@ -140,12 +142,11 @@ def poll_request(
         self.check_cancel = check_cancel_val
         version = "v" + version.split(".")[0]  # Convert SemVer into Major API version
         base_url: str = str(server) + version + endpoint
-        request_headers: dict = REQUEST_HEADERS[service]
 
         logger.info(f"Sending {operation} polling request to {base_url}. Query Parameters - {query_params}")
 
         try:
-            response = polling2.poll(lambda: requests.get(base_url, headers=request_headers, params=query_params),
+            response = polling2.poll(lambda: requests.get(base_url, headers=self.request_headers, params=query_params),
                                      step=polling_interval, timeout=polling_timeout,
                                      check_success=self.check_poll)
             return response

compliance_suite/functions/report.py

@@ -22,9 +22,9 @@ def __init__(self):
     def initialize_report(self) -> None:
         """Set Testbed Report details"""
 
-        self.report.set_testbed_name("TES Compliance Suite")
+        self.report.set_testbed_name("API Compliance Suite")
         self.report.set_testbed_version("0.1.0")
-        self.report.set_testbed_description("TES Compliance Suite tests the platform against the GA4GH TES API "
+        self.report.set_testbed_description("The compliance suite tests the platform against the GA4GH TES API "
                                             "specs. Its an automated tool system testing against YAML-based test "
                                             "files along with the ability to validate cloud service/functionality.")
 
@@ -37,7 +37,7 @@ def set_platform_details(self, platform_server: str) -> None:
 
         self.platform_name = platform_server
         self.report.set_platform_name(platform_server)
-        self.report.set_platform_description(f"TES service deployed on the {platform_server}")
+        self.report.set_platform_description(f"API service deployed on the {platform_server}")
 
     def add_phase(self, filename: str, description: str) -> Any:
         """Add a phase which is individual YAML test file

compliance_suite/job_runner.py

@@ -11,12 +11,7 @@
     List
 )
 
-from jsonschema import (
-    RefResolver,
-    validate,
-    ValidationError
-)
-import yaml
+from ga4gh.testbed.report.test import Test
 
 from compliance_suite.constants.constants import (
     PATTERN_HASH_CENTERED,
@@ -36,10 +31,10 @@
 )
 from compliance_suite.test_runner import TestRunner
 from compliance_suite.utils.test_utils import (
+    load_and_validate_yaml_data,
     replace_string,
     tag_matcher
 )
-from ga4gh.testbed.report.test import Test
 
 
 class JobRunner:
@@ -50,7 +45,7 @@ def __init__(self, server: str, version: str):
 
         Args:
             server (str): The server URL on which the compliance suite will be run
-            version (str): The compliance suite will be run against this TES version
+            version (str): The compliance suite will be run against this API version
         """
 
         self.server: str = server
@@ -115,46 +110,6 @@ def generate_summary(self) -> None:
         logger.summary("", PATTERN_HASH_CENTERED)
         logger.summary("\n\n\n")
 
-    def load_and_validate_yaml_data(self, yaml_file: str, _type: str):
-        """
-        Load and validate YAML data from the file with the provided schema type.
-
-        Args:
-            yaml_file: The path to the YAML file.
-            _type: The type of YAML file, either "Test" or "Template".
-
-        Returns:
-            The loaded and validated YAML data.
-        """
-
-        # Load YAML data
-        try:
-            yaml_data = yaml.safe_load(open(yaml_file, "r"))
-        except yaml.YAMLError as err:
-            raise JobValidationException(name="YAML Error",
-                                         message=f"Invalid YAML file {yaml_file}",
-                                         details=err)
-
-        # Validate YAML data with schema
-        schema_dir_path = Path("docs/test_config").absolute()
-        test_schema_path = Path("docs/test_config/test_schema.json")
-        template_schema_path = Path("docs/test_config/template_schema.json")
-        schema_file_path = str(test_schema_path if _type == TEST else template_schema_path)
-        json_schema = yaml.safe_load(open(schema_file_path, "r"))
-
-        try:
-            # Python-jsonschema does not reference local files directly
-            # Refer solution from https://github.com/python-jsonschema/jsonschema/issues/98#issuecomment-105475109
-            resolver = RefResolver('file:///' + str(schema_dir_path).replace("\\", "/") + '/', None)
-            validate(yaml_data, json_schema, resolver=resolver)
-            logger.info(f'YAML file valid for {_type}: {yaml_file}')
-        except ValidationError as err:
-            raise JobValidationException(name="YAML Schema Validation Error",
-                                         message=f"YAML file {yaml_file} does not match the {_type} schema",
-                                         details=err.message)
-
-        return yaml_data
-
     def generate_report(self) -> Any:
         """Generates the report via ga4gh-testbed-lib and returns it
 
@@ -178,16 +133,16 @@ def initialize_test(self, yaml_file: Path) -> None:
         logger.summary(f"     Initiating Test-{self.test_count} for {yaml_file}     ", PATTERN_HASH_CENTERED)
 
         try:
-            yaml_data = self.load_and_validate_yaml_data(str(yaml_file), TEST)
+            yaml_data = load_and_validate_yaml_data(str(yaml_file), TEST)
             report_phase = self.report.add_phase(str(yaml_file), yaml_data["description"])
 
             if (self.version in yaml_data["versions"]
                     and tag_matcher(self.include_tags, self.exclude_tags, yaml_data["tags"])):
-                test_runner = TestRunner(yaml_data["service"], self.server, self.version)
+                test_runner = TestRunner(self.server, self.version)
                 job_list: List[Dict] = []
                 for job in yaml_data["jobs"]:
                     if "$ref" in job:
-                        template_data = self.load_and_validate_yaml_data(job["$ref"], TEMPLATE)
+                        template_data = load_and_validate_yaml_data(job["$ref"], TEMPLATE)
                         if "args" in job:
                             for key, value in job["args"].items():
                                 template_data = replace_string(template_data, f"{{{key}}}", value)
@@ -233,7 +188,7 @@ def run_jobs(self) -> None:
             if search_path.is_file() and search_path.match("*.yml"):
                 self.initialize_test(search_path)
             elif search_path.is_dir():
-                for yaml_file in search_path.glob("**/*.yml"):
+                for yaml_file in sorted(search_path.glob("**/*.yml")):
                     self.initialize_test(yaml_file)
 
         self.generate_summary()