From cf38a92f28df3a28816adb5fade86e3c78051b11 Mon Sep 17 00:00:00 2001 From: Alastair Lyall Date: Fri, 19 Jul 2024 17:56:35 +0200 Subject: [PATCH] feat(gherkin): add gherkin parser for workflow testing (#464) Closes #463 Co-Authored-By: Giuseppe Steduto --- AUTHORS.md | 1 + MANIFEST.in | 1 + reana_commons/gherkin_parser/__init__.py | 9 + reana_commons/gherkin_parser/data_fetcher.py | 95 +++ reana_commons/gherkin_parser/errors.py | 24 + reana_commons/gherkin_parser/functions.py | 673 ++++++++++++++++++ reana_commons/gherkin_parser/parser.py | 259 +++++++ setup.py | 2 + tests/conftest.py | 51 ++ .../features/expected-failure.feature | 14 + .../features/failing-test.feature | 12 + .../features/log-content.feature | 26 + .../features/test-gherkin-syntax.feature | 10 + .../features/workflow-duration.feature | 14 + .../workflow-execution-completes.feature | 10 + .../features/workspace-content.feature | 20 + tests/test_gherkin_functions.py | 123 ++++ tests/test_gherkin_parser.py | 168 +++++ 18 files changed, 1512 insertions(+) create mode 100644 reana_commons/gherkin_parser/__init__.py create mode 100644 reana_commons/gherkin_parser/data_fetcher.py create mode 100644 reana_commons/gherkin_parser/errors.py create mode 100644 reana_commons/gherkin_parser/functions.py create mode 100644 reana_commons/gherkin_parser/parser.py create mode 100644 tests/gherkin_test_files/features/expected-failure.feature create mode 100644 tests/gherkin_test_files/features/failing-test.feature create mode 100644 tests/gherkin_test_files/features/log-content.feature create mode 100644 tests/gherkin_test_files/features/test-gherkin-syntax.feature create mode 100644 tests/gherkin_test_files/features/workflow-duration.feature create mode 100644 tests/gherkin_test_files/features/workflow-execution-completes.feature create mode 100644 tests/gherkin_test_files/features/workspace-content.feature create mode 100644 tests/test_gherkin_functions.py create mode 100644 tests/test_gherkin_parser.py diff --git a/AUTHORS.md b/AUTHORS.md index b4f51d10..e88ac830 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -4,6 +4,7 @@ The list of contributors in alphabetical order: - [Adelina Lintuluoto](https://orcid.org/0000-0002-0726-1452) - [Agisilaos Kounelis](https://orcid.org/0000-0001-9312-3189) +- [Alastair Lyall](https://orcid.org/0009-0000-4955-8935) - [Audrius Mecionis](https://orcid.org/0000-0002-3759-1663) - [Bruno Rosendo](https://orcid.org/0000-0002-0923-3148) - [Burt Holzman](https://orcid.org/0000-0001-5235-6314) diff --git a/MANIFEST.in b/MANIFEST.in index 01b79afc..e34f4113 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -23,3 +23,4 @@ recursive-include docs *.txt recursive-include reana_commons *.json recursive-include reana_commons *.py recursive-include tests *.py +recursive-include tests *.feature diff --git a/reana_commons/gherkin_parser/__init__.py b/reana_commons/gherkin_parser/__init__.py new file mode 100644 index 00000000..209bb686 --- /dev/null +++ b/reana_commons/gherkin_parser/__init__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2019, 2021 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""REANA-Commons gherkin parser package.""" diff --git a/reana_commons/gherkin_parser/data_fetcher.py b/reana_commons/gherkin_parser/data_fetcher.py new file mode 100644 index 00000000..735d8fa3 --- /dev/null +++ b/reana_commons/gherkin_parser/data_fetcher.py @@ -0,0 +1,95 @@ +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + + +# This file provides primitives required for gherkin_parser/functions.py, allowing for different +# implementations in both the client side (api calls) and server side (database access). This avoids +# circular dependencies between reana-commons and reana-client. +"""Base class for fetching data related to a workflow.""" + +from abc import ABC, abstractmethod + + +class DataFetcherBase(ABC): + """Base class for fetching date related to a workflow.""" + + @abstractmethod + def list_files(self, workflow, file_name=None, page=None, size=None, search=None): + """Return the list of files for a given workflow workspace. + + :param workflow: name or id of the workflow. + :param file_name: file name(s) (glob) to list. + :param page: page number of returned file list. + :param size: page size of returned file list. + :param search: filter search results by parameters. + :returns: a list of dictionaries that have the ``name``, ``size`` and + ``last-modified`` keys. + """ + pass + + @abstractmethod + def get_workflow_disk_usage(self, workflow, parameters): + """Display disk usage workflow. + + :param workflow: name or id of the workflow. + :param parameters: a dictionary to customize the response. It has the following + (optional) keys: + + - ``summarize``: a boolean value to indicate whether to summarize the response + to include only the total workspace disk usage + - ``search``: a string to filter the response by file name + + :return: a dictionary containing the ``workflow_id``, ``workflow_name``, and the ``user`` ID, with + a ``disk_usage_info`` keys that contains a list of dictionaries, each of one corresponding + to a file, with the ``name`` and ``size`` keys. + """ + pass + + @abstractmethod + def get_workflow_logs(self, workflow, steps=None, page=None, size=None): + """Get logs from a workflow engine. + + :param workflow: name or id of the workflow. + :param steps: list of step names to get logs for. + :param page: page number of returned log list. + :param size: page size of returned log list. + + :return: a dictionary with a ``logs`` key containing a JSON string that + contains the requested logs. + """ + pass + + @abstractmethod + def get_workflow_status(self, workflow): + """Get status of previously created workflow. + + :param workflow: name or id of the workflow. + :return: a dictionary with the information about the workflow status. + The dictionary has the following keys: ``id``, ``logs``, ``name``, + ``progress``, ``status``, ``user``. + """ + pass + + @abstractmethod + def get_workflow_specification(self, workflow): + """Get specification of previously created workflow. + + :param workflow: name or id of the workflow. + :returns: a dictionary that cointains two top-level keys: ``parameters``, and + ``specification`` (which contains a dictionary created from the workflow specification). + """ + pass + + @abstractmethod + def download_file(self, workflow, file_name): + """Download the requested file if it exists. + + :param workflow: name or id of the workflow. + :param file_name: file name or path to the file requested. + :return: a tuple containing file binary content, filename and whether + the returned file is a zip archive containing multiple files. + """ + pass diff --git a/reana_commons/gherkin_parser/errors.py b/reana_commons/gherkin_parser/errors.py new file mode 100644 index 00000000..530d0877 --- /dev/null +++ b/reana_commons/gherkin_parser/errors.py @@ -0,0 +1,24 @@ +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +"""Gherkin parser errors.""" + + +class StepDefinitionNotFound(Exception): + """The step definition was not found.""" + + pass + + +class StepSkipped(Exception): + """The step was skipped.""" + + pass + + +class FeatureFileError(Exception): + """The feature file is invalid.""" + + pass diff --git a/reana_commons/gherkin_parser/functions.py b/reana_commons/gherkin_parser/functions.py new file mode 100644 index 00000000..ee035033 --- /dev/null +++ b/reana_commons/gherkin_parser/functions.py @@ -0,0 +1,673 @@ +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + + +# This script contains the step definitions for the Gherkin feature file tests. +# +# It should be not run directly. Rather, the functions defined here should be imported and +# mapped to the corresponding steps in the feature file. +# +# An example of this is done in the `parse_and_run_tests` function of the +# `gherkin_parser/parser.py` method. + +# This module is not commented using a docstring to avoid it becoming part of the Sphinx documentation page. +# noqa: D100 + +import hashlib +import json +import re +import zlib +import inspect +from functools import partial, update_wrapper +from typing import List, Dict +from reana_commons.gherkin_parser.errors import StepSkipped +from reana_commons.gherkin_parser.data_fetcher import DataFetcherBase +from datetime import datetime +from reana_commons.config import WORKFLOW_TIME_FORMAT as DATETIME_FORMAT + + +def given(step_pattern): + """Decorate to mark a function as a given step.""" + + def wrapper(func): + func._step_keyword = "given" + func._step_type = "Context" + try: + func._step_pattern.append(step_pattern) + except Exception: + func._step_pattern = [step_pattern] + return func + + return wrapper + + +def when(step_pattern): + """Decorate to mark a function as a given step.""" + + def wrapper(func): + func._step_keyword = "when" + func._step_type = "Action" + try: + func._step_pattern.append(step_pattern) + except Exception: + func._step_pattern = [step_pattern] + return func + + return wrapper + + +def then(step_pattern): + """Decorate to mark a function as a given step.""" + + def wrapper(func): + func._step_keyword = "then" + func._step_type = "Outcome" + try: + func._step_pattern.append(step_pattern) + except Exception: + func._step_pattern = [step_pattern] + + return func + + return wrapper + + +def _get_step_definition_lists(fetcher: DataFetcherBase): + """Return a dictionary containing the lists of all the step definitions in this module. + + The dictionary has three keys: "Outcome", "Action", and "Context". + Each key contains a list of functions (step definitions) for that step type. + """ + step_definition_list = { + "Outcome": [], + "Action": [], + "Context": [], + } + for type in step_definition_list.keys(): + step_definition_list[type] = [ + ( + update_wrapper(partial(f, data_fetcher=fetcher), f) + if "data_fetcher" in (inspect.signature(f).parameters) + else f + ) + for f in globals().values() + if callable(f) and hasattr(f, "_step_type") and (f._step_type == type) + ] + return step_definition_list + + +def _get_single_file_size(workflow, filename, data_fetcher) -> Dict: + """Return a dictionary with the size of a single file. + + The dictionary has two keys: "raw" and "human_readable", which contain the size + in bytes and in human-readable format, respectively. + """ + files_info = data_fetcher.list_files(workflow, file_name=filename) + if len(files_info) == 1: + return files_info[0]["size"] + else: + raise Exception( + f"The specified file name ({filename}) is not in the workspace!" + ) + + +def _get_total_workspace_size(workflow, data_fetcher) -> int: + """Return the total workspace size, as raw number of bytes.""" + disk_usage = data_fetcher.get_workflow_disk_usage(workflow, {"summarize": True})[ + "disk_usage_info" + ][0] + return disk_usage["size"]["raw"] + + +def _human_readable_to_raw(dim: str) -> int: + """Convert the size to the raw number of bytes, whether it's in a human-readable format or not. + + Allowed formats are: + - raw number of bytes (e.g. 1024) + - human-readable units B/KiB/MiB/GiB/TiB/PiB (e.g. 1 KiB, 1MiB, 1.5GiB, 1TiB, 1PiB) + - "bytes" suffix (e.g. 1234 bytes, 156632 bytes) + The conversion is carried out considering multiples of 1024, as per the IEC standard. + + :param dim: The string that represents the size (in human-readable format or raw) + :return: The equivalent number of bytes + """ + # Conversion factors + units = { + "": 1, + "bytes": 1, + "B": 1, + "KiB": 1024, + "MiB": 1024**2, + "GiB": 1024**3, + "TiB": 1024**4, + "PiB": 1024**5, + } + + # Regex pattern to extract size and unit + pattern = r"(\d+(\.\d+)?)\s*([A-Za-z]*)" + match = re.match(pattern, dim) + + if match: + size = float(match.group(1)) + unit = match.group(3) + + # Ensure the unit is supported + if unit in units: + return int(size * units[unit]) + else: + raise ValueError(f'Unknown unit "{unit}"') + + raise ValueError(f'Unable to parse "{dim}"') + + +def _is_file_in_workspace(workflow: str, filename: str, data_fetcher) -> bool: + """Check whether a file is in the workspace of the workflow.""" + parameters = {"summarize": False} + disk_usage_info = data_fetcher.get_workflow_disk_usage(workflow, parameters)[ + "disk_usage_info" + ] + workflow_files = [file["name"] for file in disk_usage_info] + # When checking, try to add a trailing slash, in case the user forgot it + return filename in workflow_files or f"/{filename}" in workflow_files + + +def _strip_quotes(string: str) -> str: + """Delete the leading and trailing quotes from a string, if present. + + :param string: + :return: The string without quotes + """ + return string.strip('"') + + +def _remove_prefix(string: str, prefix: str) -> str: + """Remove the prefix from the string, if present. + + :param string: The string from which the prefix has to be removed + :param prefix: The prefix to be removed + """ + if string.startswith(prefix): + return string[len(prefix) :] + return string + + +def _remove_prefixes(string: str, prefixes: List[str]) -> str: + """Remove any of the prefix in the arguments, if present. + + :param string: The string from which the prefix has to be removed + :param prefixes: The list of prefixes to be removed + """ + for prefix in prefixes: + string = _remove_prefix(string, prefix) + return string + + +def _job_logs_contain(workflow, content, data_fetcher): + log_data = data_fetcher.get_workflow_logs(workflow)["logs"] + job_logs = json.loads(log_data)["job_logs"] + for step_info in job_logs.values(): + if content in step_info["logs"]: + return True + return False + + +def _engine_logs_contain(workflow, content, data_fetcher): + log_data = data_fetcher.get_workflow_logs(workflow)["logs"] + engine_log = (json.loads(log_data)["workflow_logs"] or "") + ( + json.loads(log_data)["engine_specific"] or "" + ) + return content in engine_log + + +@when("the workflow execution completes") +def _check_workflow_finished(workflow, data_fetcher): + """ + .. container:: testcase-title. + + Workflow execution completed + + | ``When the workflow execution completes`` + + The tests in this scenario will run only if the workflow has completed its execution (regardless of whether + the execution was successful or not). + """ + response = data_fetcher.get_workflow_status(workflow) + if response["status"] not in [ + "finished", + "failed", + ]: + raise StepSkipped( + f'The execution of the workflow "{workflow}" has not completed yet. Its status is "{response["status"]}"' + ) + + +@when("the workflow is {status_workflow}") +@when("the workflow status is {status_workflow}") +def _check_workflow_status(workflow, status_workflow, data_fetcher): + """ + .. container:: testcase-title. + + Status of the workflow + + | ``When the workflow status is {status_workflow}`` + + :param status_workflow: The status in which the workflow run has to be in order to run the tests from the scenario + + The tests in this scenario will run only if the workflow is in the specified status. + """ + status_workflow = _strip_quotes(status_workflow) + response = data_fetcher.get_workflow_status(workflow) + if response["status"] != status_workflow: + raise StepSkipped( + f'The workflow "{workflow}" is not "{status_workflow}" status. Its status is "{response["status"]}".' + ) + + +@then("the workflow should be {status_workflow}") +@then("the workflow status should be {status_workflow}") +def _test_workflow_status(workflow, status_workflow, data_fetcher): + """ + .. container:: testcase-title. + + Status of the workflow + + | ``Then the workflow should be {status_workflow}`` + | ``Then the workflow status should be {status_workflow}`` + + :param status_workflow: The status in which the workflow run has to be + + This test will pass only if the workflow is in the specified status. + """ + status_workflow = _strip_quotes(status_workflow) + response = data_fetcher.get_workflow_status(workflow) + assert ( + response["status"] == status_workflow + ), f'The workflow "{workflow}" is not "{status_workflow}". Its status is "{response["status"]}".' + + +@then("the outputs should be included in the workspace") +@then("all the outputs should be included in the workspace") +def workspace_include_all_outputs(workflow, data_fetcher): + """ + .. container:: testcase-title. + + Presence of all the outputs in the workspace + + | ``Then all the outputs should be included in the workspace`` + | ``Then the outputs should be included in the workspace`` + + This test will pass only if the workspace contains all the files and directories + specified under the "output" section of the REANA specification file. + """ + specs = data_fetcher.get_workflow_specification(workflow) + spec_outputs = specs.get("specification", {}).get("outputs", {}) + outputs = ( + spec_outputs.get("files", []) or [] + spec_outputs.get("directories", []) or [] + ) + for filename in outputs: + assert ( + _is_file_in_workspace(workflow, filename, data_fetcher) is True + ), f'The workspace does not contain "{filename}"!' + + +@then('the workspace should include "{filename}"') +@then('the workspace should contain "{filename}"') +@then("{filename} should be in the workspace") +def workspace_include_specific_file(workflow, filename, data_fetcher): + """ + .. container:: testcase-title. + + Presence of a specific file in the workspace + + | ``Then the workspace should include {filename}`` + | ``Then the workspace should contain {filename}`` + | ``Then {filename} should be in the workspace`` + + :param filename: The path of the file in the workspace. + + This test will pass only if the workspace contains ``{filename}``. + """ + assert ( + _is_file_in_workspace(workflow, filename, data_fetcher) is True + ), f'The workspace does not contain "{filename}"!' + + +@then("the workspace should not include {filename}") +@then("the workspace should not contain {filename}") +@then("{filename} should not be in the workspace") +def workspace_do_not_include_specific_file(workflow, filename, data_fetcher): + """ + .. container:: testcase-title. + + Absence of a specific file in the workspace + + | ``Then the workspace should not include {filename}`` + | ``Then the workspace should not contain {filename}`` + | ``Then {filename} should not be in the workspace`` + + :param filename: The path of the file in the workspace. + + This test will pass only if the workspace does not contain ``{filename}``. + """ + assert ( + _is_file_in_workspace(workflow, filename, data_fetcher) is False + ), f'The workspace contains "{filename}"!' + + +@then('the logs should contain "{content}"') +def logs_contain(workflow, content, data_fetcher): + """ + .. container:: testcase-title. + + Content of the logs of the workflow + + | ``Then the logs should contain "{content}"`` + + :param content: The content that should be inside the logs. Note that this parameter + MUST be surrounded by quotes. + + This test will pass only if the logs (either engine or job logs) contain ``{content}``. + """ + assert _engine_logs_contain(workflow, content, data_fetcher) or _job_logs_contain( + workflow, content, data_fetcher + ), f'The logs do not contain "{content}"!' + + +@then('the engine logs should contain "{content}"') +def logs_engine_contain(workflow, content, data_fetcher): + """ + .. container:: testcase-title. + + Content of the engine logs of the workflow + + | ``Then the engine logs should contain "{content}"`` + + :param content: The content that should be inside the engine logs. Note that this parameter + MUST be surrounded by quotes. + + This test will pass only if the engine logs contain ``{content}``. + """ + assert _engine_logs_contain( + workflow, content, data_fetcher + ), f'The engine logs do not contain "{content}"!' + + +@then('the job logs should contain "{content}"') +def logs_job_contain(workflow, content, data_fetcher): + """ + .. container:: testcase-title. + + Content of the job logs of the workflow + + | ``Then the job logs should contain "{content}"`` + + :param content: The content that should be inside the engine logs. Note that this parameter + MUST be surrounded by quotes. + + This test will pass only if the job logs contain ``{content}``. + """ + assert _job_logs_contain( + workflow, content, data_fetcher + ), f'The job logs do not contain "{content}"!' + + +@then('the job logs for the step {step_name} should contain "{content}"') +@then('the job logs for the {step_name} step should contain "{content}"') +def logs_step_contain(workflow, step_name, content, data_fetcher): + """ + .. container:: testcase-title. + + Content of the job logs for a particular step + + | ``Then the job logs for the step "{step_name}" should contain "{content}"`` + | ``Then the job logs for the "{step_name}" step should contain "{content}"`` + + :param step_name: The name of the step whose logs have to be checked. + :param content: The content that should be inside the logs. Note that this parameter + MUST be surrounded by quotes. + + This test will pass only if the job logs relative to the step ``{step_name}`` contain ``{content}``. + """ + step_name = _strip_quotes(step_name) + logs_contain(workflow, content, data_fetcher) + try: + _, logs_for_step = json.loads( + data_fetcher.get_workflow_logs(workflow, steps=[step_name])["logs"] + )["job_logs"].popitem() + except KeyError: + # The dictionary is empty, thus the step name is invalid + raise Exception("The specified step name is invalid!") + assert ( + content in logs_for_step["logs"] + ), f'The logs for the step "{step_name}" do not contain the specified content "{content}". Logs: "{logs_for_step["logs"]}"' + + +@then('the file {filename} should include "{content}"') +@then('the file {filename} should contain "{content}"') +def file_content_contain(workflow, filename, content, data_fetcher): + """ + .. container:: testcase-title. + + Content of a specific file in the workspace + + | ``Then the file {filename} should include "{content}"`` + | ``Then the file {filename} should include "{content}"`` + + :param filename: The path of the file in the workspace. + :param content: The content that should be inside the file. Note that this parameter + MUST be surrounded by quotes. + + This test will pass only if the specified file (which has to be in the workspace) + contains the required ``{content}``. + """ + filename = _strip_quotes(filename) + if not filename.startswith("/"): + filename = f"/{filename}" # Add leading slash if needed + (file_content, file_name, is_archive) = data_fetcher.download_file( + workflow, filename + ) + if is_archive: + raise StepSkipped("This test is not supported for archive files.") + assert content in file_content.decode( + "utf-8" + ), f'The file does not contain "{content}"!' + + +@then("the {algorithm} checksum of the file {filename} should be {checksum}") +def file_checksum(workflow, algorithm, filename, checksum, data_fetcher): + """ + .. container:: testcase-title. + + Checksum of a specific file in the workspace + + | ``Then the {algorithm} checksum of the file {filename} should be {checksum}`` + + :param algorithm: The algorithm to use for the checksum. Algorithms supported: ``sha256``, ``sha512``, ``md5``, ``adler32``. + :param filename: The path of the file in the workspace. + :param checksum: The expected checksum of the file. + + This test will pass only if the specified file (which has to be in the workspace) + has the required ``{checksum}``. + """ + filename = _strip_quotes(filename) + checksum = _strip_quotes(checksum) + algorithm = _strip_quotes(algorithm) + supported_algorithms = {"sha256", "sha512", "md5", "adler32"} + if algorithm.lower() not in supported_algorithms: + raise Exception( + f"The specified checksum algorithm is not supported! Supported algorithms: {supported_algorithms}" + ) + (file_content, file_name, is_archive) = data_fetcher.download_file( + workflow, filename + ) + # Checksum the file content + if algorithm.lower() == "adler32": + h = zlib.adler32(file_content) + computed_hash = hex(h)[2:] + checksum = _remove_prefix(checksum.lower(), "0x") + else: + h = hashlib.new(algorithm.upper()) + h.update(file_content) + computed_hash = h.hexdigest() + assert ( + computed_hash == checksum + ), f'The checksum of the file "{filename}" is not "{checksum}"! Actual checksum: "{computed_hash}"' + + +@then("the workflow run duration should be less than {n_minutes} minutes") +def duration_minimum_workflow(workflow, n_minutes, data_fetcher): + """ + .. container:: testcase-title. + + Minimum duration of the workflow + + | ``Then the workflow run duration should be less than {n_minutes} minutes`` + + :param n_minutes: The maximum duration, in minutes, of the analysis run. + + This test will pass only if the analysis run took less than ``{n_minutes}`` minutes. + """ + n_minutes = _strip_quotes(n_minutes) + status = data_fetcher.get_workflow_status(workflow) + run_finished_at = datetime.strptime( + status["progress"]["run_finished_at"], DATETIME_FORMAT + ) + run_started_at = datetime.strptime( + status["progress"]["run_started_at"], DATETIME_FORMAT + ) + duration = (run_finished_at - run_started_at).total_seconds() + assert duration / 60 < float( + n_minutes + ), f"The workflow took more than {n_minutes} minutes to complete! Run duration: {duration / 60} minutes" + + +@then("the duration of the step {step_name} should be less than {n_minutes} minutes") +def duration_minimum_step(workflow, step_name, n_minutes, data_fetcher): + """ + .. container:: testcase-title. + + Minimum duration of each step of the workflow + + | ``Then the duration of the step {step_name} should be less than {n_minutes} minutes`` + + :param step_name: The name of the step whose duration has to be checked. + :param n_minutes: The maximum duration, in minutes, of the analysis run. + + This test will pass only if the step ``{step_name}`` of the analysis took less than + ``{n_minutes}`` minutes. + """ + step_name = _strip_quotes(step_name) + n_minutes = _strip_quotes(n_minutes) + _, logs_for_step = json.loads( + data_fetcher.get_workflow_logs(workflow, steps=[step_name])["logs"] + )["job_logs"].popitem() + try: + _, logs_for_step = json.loads( + data_fetcher.get_workflow_logs(workflow, steps=[step_name])["logs"] + )["job_logs"].popitem() + except KeyError: + # The dictionary is empty, thus the step name is invalid + raise Exception("The specified step name is invalid!") + + duration = ( + datetime.strptime(logs_for_step["finished_at"], DATETIME_FORMAT) + - datetime.strptime(logs_for_step["started_at"], DATETIME_FORMAT) + ).total_seconds() + assert duration / 60 < float( + n_minutes + ), f"The step took more than {n_minutes} minutes to complete! Run duration: {duration / 60} minutes" + + +@then("the size of the file {filename} should be exactly {dim}") +def exact_file_size(workflow, filename, dim, data_fetcher): + """ + .. container:: testcase-title. + + Exact size of a specific file in the workspace + + | ``Then the size of the file {filename} should be {dim} bytes`` + + :param filename: The path of the file in the workspace. + :param dim: The size of the file. This parameter can be expressed either as raw number of bytes or in a human-readable format (e.g. ``1.5 GiB``). + + This test will pass only if the specified file (which has to be in the workspace) + has the required size ``{dim}``. + """ + dim = _human_readable_to_raw(_strip_quotes(dim)) + filename = _remove_prefixes(_strip_quotes(filename), ["./", "/"]) + file_size_dict = _get_single_file_size(workflow, filename, data_fetcher) + assert ( + file_size_dict["raw"] == dim + ), f'The size of the file "{filename}" is not {dim}! Actual size: {file_size_dict["raw"]} bytes ({file_size_dict["human_readable"]})' + + +@then("the size of the file {filename} should be between {dim1} and {dim2}") +def approximate_file_size(workflow, filename, dim1, dim2, data_fetcher): + """ + .. container:: testcase-title. + + Approximate size of a specific file in the workspace + + | ``Then the size of the file be between {dim1} and {dim2}`` + | ``Then the size of the file be between {dim1} and {dim2} bytes`` + + :param filename: The path of the file in the workspace. + :param dim1: The lower bound of the file size. This parameter can be expressed as raw number of bytes (e.g. 14336) or as a human-readable string (e.g. 14KiB). + :param dim2: The upper bound of the file size. This parameter can be expressed as raw number of bytes (e.g. 14336) or as a human-readable string (e.g. 14KiB). + + This test will pass only if the specified file (which has to be in the workspace) + has a size between ``{dim1}`` and ``{dim2}``. + """ + dim1_raw = _human_readable_to_raw(_strip_quotes(dim1)) + dim2_raw = _human_readable_to_raw(_strip_quotes(dim2)) + # Reorder the dimensions if needed + if dim1_raw > dim2_raw: + dim1_raw, dim2_raw = dim2_raw, dim1_raw + filename = _remove_prefixes(_strip_quotes(filename), ["./", "/"]) + file_size_dict = _get_single_file_size(workflow, filename, data_fetcher) + assert ( + dim1_raw <= file_size_dict["raw"] <= dim2_raw + ), f'The size of the file "{filename}" is not between {dim1} and {dim2} bytes! Actual size: {file_size_dict["raw"]} bytes ({file_size_dict["human_readable"]}).' + + +@then("the workspace size should be less than {dim}") +def workspace_size_maximum(workflow, dim, data_fetcher): + """ + .. container:: testcase-title. + + Maximum workspace size + + | ``Then the workspace size should be less than {dim}`` + + :param dim: The maximum size of the workspace. This parameter may be expressed either as raw number of bytes (e.g. 14000), or as a human-readable string, in the "X YiB" format (e.g. 15 MiB) + + This test will pass only if the total size of the workspace is less than ``{dim}``. + """ + dim_raw = _human_readable_to_raw(_strip_quotes(dim)) + workspace_dim = _get_total_workspace_size(workflow, data_fetcher) + assert ( + workspace_dim <= dim_raw + ), f"The workspace size is more than {dim}! Workspace size: {workspace_dim} bytes" + + +@then("the workspace size should be more than {dim}") +def workspace_size_minimum(workflow, dim, data_fetcher): + """ + .. container:: testcase-title. + + Minimum workspace size + + | ``Then the workspace size should be more than {dim}`` + + :param dim: The minimum size of the workspace. This parameter may be expressed either as raw number of bytes (e.g. 14000), or as a human-readable string, in the "X YiB" format (e.g. 15 MiB) + + This test will pass only if the total size of the workspace is more than ``{dim}``. + """ + dim_raw = _human_readable_to_raw(_strip_quotes(dim)) + workspace_dim = _get_total_workspace_size(workflow, data_fetcher) + assert ( + workspace_dim >= dim_raw + ), f"The workspace size is less than {dim}! Workspace size: {workspace_dim} bytes" diff --git a/reana_commons/gherkin_parser/parser.py b/reana_commons/gherkin_parser/parser.py new file mode 100644 index 00000000..2d017714 --- /dev/null +++ b/reana_commons/gherkin_parser/parser.py @@ -0,0 +1,259 @@ +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +"""Gherkin test runner.""" + +import logging +import enum +from datetime import datetime, timezone +from typing import Dict, List, Tuple +from gherkin.parser import Parser +from gherkin.pickles.compiler import Compiler +from parse import parse +from dataclasses import dataclass +from reana_commons.gherkin_parser.errors import ( + StepDefinitionNotFound, + StepSkipped, + FeatureFileError, +) +from reana_commons.gherkin_parser.functions import _get_step_definition_lists +from reana_commons.gherkin_parser.data_fetcher import DataFetcherBase + + +class AnalysisTestStatus(enum.Enum): + """Enumeration of possible analysis test statuses.""" + + passed = 0 + failed = 1 + skipped = 2 + + +@dataclass +class TestResult: + """Dataclass for storing test results.""" + + scenario: str + failed_testcase: str + result: AnalysisTestStatus + error_log: str + feature: str + checked_at: datetime + + +def _get_step_text(step: Dict) -> str: + """Return the text of the step, including possible multiline arguments. + + :param step: A step dictionary, as returned by `parse_feature_file`. + :return: The text of the step, including possible multiline arguments. + """ + if "argument" in step.keys(): + return f'{step["text"]} "{step["argument"]["docString"]["content"]}"' + return step["text"] + + +def validate_feature_file(feature_file_path: str, data_fetcher: DataFetcherBase): + """Validate the feature file. + + :param feature_file_path: The path to the feature file to be validated. + :return A tuple containing the feature name, the parsed feature object and the dictionary mapping + the step texts to their function definitions. + :raise StepDefinitionNotFound: If the feature file contains a step for which no step definition is found. + :raise FeatureFileError: If there is an error while parsing or compiling the feature file. + :raise FileNotFoundError: If the feature file does not exist. + :raise IOError: If there is an error while reading the feature file. + """ + feature_name, parsed_feature = parse_feature_file(feature_file_path) + steps_list = get_steps_list(parsed_feature) + step_map = map_steps_to_functions(steps_list, data_fetcher) + return feature_name, parsed_feature, step_map + + +def parse_feature_file(feature_file_path: str) -> Tuple[str, List[Dict]]: + """Parse the feature file and return the list of scenarios. + + :param feature_file_path: The path to the feature file to be parsed. + :return: A tuple containing the feature name and a list of dictionaries, + each corresponding to one scenario of the feature file. + Example: + { + "name": "Scenario name", + "steps": List of steps, each containing at least the step type and the step text. + } + :raise FileNotFoundError: If the feature file does not exist. + :raise IOError: If there is an error while reading the feature file. + :raise FeatureFileError: If there is an error while parsing or compiling the feature file. + """ + try: + with open(feature_file_path) as f: + file_content = f.read() + except FileNotFoundError as e: + raise FileNotFoundError( + f"The file '{feature_file_path}' was not found: {e.strerror}" + ) + except IOError as e: + raise IOError(f"Error reading the file '{feature_file_path}': {e.strerror}") + + try: + parser = Parser() + compiler = Compiler() + gherkin_document = parser.parse(file_content) + feature_name = gherkin_document["feature"]["name"] + gherkin_document["uri"] = feature_file_path + pickles = compiler.compile(gherkin_document) + return feature_name, pickles + except Exception as e: + raise FeatureFileError( + f"Unexpected error during parsing or compiling of the test file '{feature_file_path}' \n{e}" + ) + + +def get_steps_list(feature_file: List[Dict]) -> List[Tuple[str, str]]: + """Return a list of all steps in the feature file. + + :param feature_file: The parsed feature file, as returned by `parse_feature_file`. + :return: A list of tuples, each containing the step type and the step text. + The step type can be one of "Context" (Given), "Action" (When), or "Outcome" (Then). + The step text returned by this method contains also the multiline arguments, if any. + Example: + [ + ("Context", "I have a file"), + ("Action", "I run the analysis"), + ("Outcome", "the log should contain 'blah' ") + ("Outcome", "the log should contain 'bleh' ") + ] + """ + steps = [] + for scenario in feature_file: + for step in scenario["steps"]: + steps.append((step["type"].strip(), _get_step_text(step))) + return steps + + +def map_steps_to_functions(steps: list, data_fetcher: DataFetcherBase): + """Map each step to the corresponding step definition (function). + + :param steps: A list of tuples, each containing the step type, and the step text. + :return: A dictionary mapping each step to the corresponding step definition (function). + Example: + { + "Action": { + "The workflow status is finished": { + "function": , + "arguments": {"status_workflow": "finished"} + } + }, + ..., + } + :raise StepDefinitionNotFound: If the feature file contains a step for which no step definition is found. + """ + step_mapping = {"Context": {}, "Action": {}, "Outcome": {}} + for step_type, step_text in steps: + found = False + # Get the list of all step definitions, divided by step type. + step_definitions = _get_step_definition_lists(data_fetcher) + for func in step_definitions[step_type]: + # Check if the step text matches any of the patterns. + parse_results = [ + parse(pattern, step_text) for pattern in func._step_pattern + ] + result = next((r for r in parse_results if r is not None), None) + # Check if there was at least one non-None result. + if result is not None: + step_mapping[step_type][step_text] = {} + step_mapping[step_type][step_text]["function"] = func + step_mapping[step_type][step_text]["arguments"] = result.named + found = True + break + if not found: + logging.error(f"No step definition found for step: {step_text}") + raise StepDefinitionNotFound( + f"No step definition found for step: {step_text}" + ) + return step_mapping + + +def run_tests( + workflow: str, + feature_name: str, + feature_file, + step_mapping: Dict, +) -> List[TestResult]: + """Run all the tests in the parsed feature file. + + :param feature_name: The name of the feature inside the feature file. + :param workflow: The name of the workflow in REANA + :param feature_file: The parsed and compiled feature file, as returned by `parse_feature_file`. + :param step_mapping: A dictionary mapping each step to the corresponding step definition (function). + :return: A list of dictionaries, each corresponding to one scenario of the feature file. + """ + test_results = [] + for scenario in feature_file: + logging.info(f"Running scenario: {scenario['name']}...") + result = AnalysisTestStatus.passed + failed_testcase = None + error_log = None + for step in scenario["steps"]: + logging.debug(f"Running step: {step['text']} ({step['type']})...") + step_type = step["type"].strip() + step_text = _get_step_text(step) + function = step_mapping[step_type].get(step_text).get("function") + arguments = step_mapping[step_type].get(step_text).get("arguments") + if function is not None: + try: + function(workflow, **arguments) + except StepSkipped: + logging.info(f"Scenario skipped! Current testcase: {step_text}") + result = AnalysisTestStatus.skipped + break + except Exception as e: + # Catches all exceptions, including AssertionError. + result = AnalysisTestStatus.failed + failed_testcase = step_text + error_log = str(e) + logging.error(f"Scenario failed! Failed testcase: {step_text}") + logging.error(f"Error log: {e}") + break + test_results.append( + TestResult( + scenario=scenario["name"], + failed_testcase=failed_testcase, + result=result, + error_log=error_log, + feature=feature_name, + checked_at=datetime.now(timezone.utc), + ) + ) + if result == AnalysisTestStatus.passed: + logging.info(f"Scenario `{scenario['name']}` passed!") + + return test_results + + +def parse_and_run_tests( + feature_file_path: str, + workflow: str, + data_fetcher: DataFetcherBase, +) -> Tuple[str, List]: + """Parse the feature file and run all the tests in it. + + :param workflow: The name of the workflow on REANA. + :param feature_file_path: The path to the feature file to be parsed. + :return: A tuple in which the first element is the feature name, and the second is + a list of dictionaries, each corresponding to the test result of one scenario of the feature file. + :raise StepDefinitionNotFound: If the feature file contains a step for which no step definition is found. + :raise FeatureFileError: If there is an error while parsing or compiling the feature file. + :raise FileNotFoundError: If the feature file does not exist. + :raise IOError: If there is an error while reading the feature file. + """ + feature_name, parsed_feature, step_mapping = validate_feature_file( + feature_file_path, data_fetcher + ) + results = run_tests( + workflow=workflow, + feature_name=feature_name, + feature_file=parsed_feature, + step_mapping=step_mapping, + ) + return feature_name, results diff --git a/setup.py b/setup.py index f3c8cf9e..129f906d 100755 --- a/setup.py +++ b/setup.py @@ -74,6 +74,8 @@ "PyYAML>=5.1,<7.0", "Werkzeug>=0.14.1", "wcmatch>=8.3,<8.5", + "gherkin-official>=24.1.0", + "parse>=1.19.0", ] packages = find_packages() diff --git a/tests/conftest.py b/tests/conftest.py index b14231b1..1a57c328 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,8 @@ """Pytest configuration for REANA-Commons.""" import pytest +from unittest.mock import create_autospec +from reana_commons.gherkin_parser.data_fetcher import DataFetcherBase @pytest.fixture() @@ -59,3 +61,52 @@ def dummy_snakefile(): shell: "mkdir -p results && touch {output}" """ + + +@pytest.fixture() +def mock_data_fetcher(): + """Mock data fetcher for gherkin_parser tests.""" + mock_data_fetcher = create_autospec(DataFetcherBase) + mock_data_fetcher.get_workflow_status.return_value = { + "logs": { + "step-id-1": { + "job_name": "jobname", + "status": "finished", + "started_at": "2018-10-29T12:51:04", + "finished_at": "2018-10-29T12:51:37", + } + }, + "name": "test_workflow", + "progress": { + "run_started_at": "2018-10-29T12:51:04", + "run_finished_at": "2018-10-29T12:55:01", + }, + "created": "2018-10-29T12:51:04", + "status": "finished", + "user": "00000000-0000-0000-0000-000000000000", + } + mock_data_fetcher.get_workflow_disk_usage.return_value = { + "disk_usage_info": [ + { + "name": "output1.png", + "size": {"human_readable": "12 MiB", "raw": 12580000}, + }, + { + "name": "output/data.txt", + "size": {"human_readable": "100 KiB", "raw": 184320}, + }, + { + "name": "input.txt", + "size": {"human_readable": "12 MiB", "raw": 12580000}, + }, + {"name": "", "size": {"human_readable": "24 MiB", "raw": 25344320}}, + ] + } + mock_data_fetcher.get_workflow_logs.return_value = { + "logs": '{"engine_specific": "", "workflow_logs": "This is the workflow engine log output.And\\nthis\\nis a\\nmultiline string", "job_logs": {"job-id-1": {"name": "job-name-1", "logs": "Job logs of the job 1", "started_at": "2018-10-29T12:51:04", "finished_at": "2018-10-29T12:51:37"}, "job-id-2": {"name": "job-name-2", "logs": "Job logs of the job 2", "finished_at": "2018-10-29T12:55:01", "started_at": "2018-10-29T12:51:38"}}}' + } + mock_data_fetcher.get_workflow_specification.return_value = { + "specification": {"outputs": {"files": ["output1.png", "output/data.txt"]}} + } + + return mock_data_fetcher diff --git a/tests/gherkin_test_files/features/expected-failure.feature b/tests/gherkin_test_files/features/expected-failure.feature new file mode 100644 index 00000000..be460024 --- /dev/null +++ b/tests/gherkin_test_files/features/expected-failure.feature @@ -0,0 +1,14 @@ +Feature: Expected failure + Scenario: Only test status + When the workflow execution completes + Then the workflow status should be "failed" + + Scenario: Test status and a successful test + When the workflow execution completes + Then the workflow status should be "failed" + And the workspace size should be less than 250GiB + + Scenario: Test status and a successful test, different order + When the workflow execution completes + Then the workspace size should be less than 250GiB + And the workflow status should be "failed" \ No newline at end of file diff --git a/tests/gherkin_test_files/features/failing-test.feature b/tests/gherkin_test_files/features/failing-test.feature new file mode 100644 index 00000000..d9a1fd7b --- /dev/null +++ b/tests/gherkin_test_files/features/failing-test.feature @@ -0,0 +1,12 @@ +Feature: One failing test + Scenario: This test will succeed + When the workflow is finished + Then the workspace should include "output1.png" + + Scenario: This test will fail + When the workflow is finished + Then the engine logs should contain "something that is not there" + + Scenario: This test will succeed again + When the workflow is finished + Then the workspace should include "output1.png" \ No newline at end of file diff --git a/tests/gherkin_test_files/features/log-content.feature b/tests/gherkin_test_files/features/log-content.feature new file mode 100644 index 00000000..52554e38 --- /dev/null +++ b/tests/gherkin_test_files/features/log-content.feature @@ -0,0 +1,26 @@ + +# This file is part of REANA. +# Copyright (C) 2023 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +Feature: test workflow + Scenario: Log content + When the workflow is finished + Then the logs should contain "log output" + + Scenario: Engine log content + When the workflow is finished + Then the engine logs should contain "workflow engine log output" + And the engine logs should contain + """ + And + this + is a + multiline + """ + + Scenario: Job log content + When the workflow is finished + Then the job logs should contain "Job logs" \ No newline at end of file diff --git a/tests/gherkin_test_files/features/test-gherkin-syntax.feature b/tests/gherkin_test_files/features/test-gherkin-syntax.feature new file mode 100644 index 00000000..867f8911 --- /dev/null +++ b/tests/gherkin_test_files/features/test-gherkin-syntax.feature @@ -0,0 +1,10 @@ +Feature: Test Feature + Scenario: scenario 1 + Given this is a context clause + When this is an action clause + Then this is an outcome clause + And this is another outcome clause + + Scenario: scenario 2 + When the workflow is finished + Then this should be tested \ No newline at end of file diff --git a/tests/gherkin_test_files/features/workflow-duration.feature b/tests/gherkin_test_files/features/workflow-duration.feature new file mode 100644 index 00000000..41b76721 --- /dev/null +++ b/tests/gherkin_test_files/features/workflow-duration.feature @@ -0,0 +1,14 @@ +# This file is part of REANA. +# Copyright (C) 2023 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +Feature: test workflow + Scenario: Run duration + When the workflow is finished + Then the workflow run duration should be less than 5 minutes + + Scenario: Job run duration + When the workflow is finished + Then the duration of the step "jobname" should be less than 5 minutes \ No newline at end of file diff --git a/tests/gherkin_test_files/features/workflow-execution-completes.feature b/tests/gherkin_test_files/features/workflow-execution-completes.feature new file mode 100644 index 00000000..119d4779 --- /dev/null +++ b/tests/gherkin_test_files/features/workflow-execution-completes.feature @@ -0,0 +1,10 @@ +# This file is part of REANA. +# Copyright (C) 2023 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +Feature: test workflow + Scenario: + When the workflow execution completes + Then the workflow status should be finished \ No newline at end of file diff --git a/tests/gherkin_test_files/features/workspace-content.feature b/tests/gherkin_test_files/features/workspace-content.feature new file mode 100644 index 00000000..1c837753 --- /dev/null +++ b/tests/gherkin_test_files/features/workspace-content.feature @@ -0,0 +1,20 @@ +# This file is part of REANA. +# Copyright (C) 2023 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +Feature: test workflow + Scenario: All the outputs in the workspace + When the workflow is finished + Then all the outputs should be included in the workspace + + Scenario: Specific files in the workspace + When the workflow is finished + Then the workspace should contain "output1.png" + And the workspace should not contain "a png file.png" + + Scenario: Workspace size + When the workflow is finished + Then the workspace size should be more than 20 MiB + And the workspace size should be less than 35344320 \ No newline at end of file diff --git a/tests/test_gherkin_functions.py b/tests/test_gherkin_functions.py new file mode 100644 index 00000000..ab9765f8 --- /dev/null +++ b/tests/test_gherkin_functions.py @@ -0,0 +1,123 @@ +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +import pytest +import os +from reana_commons.gherkin_parser.parser import parse_and_run_tests, AnalysisTestStatus +from reana_commons.gherkin_parser.functions import _human_readable_to_raw + +current_dir = os.path.dirname(os.path.abspath(__file__)) + + +@pytest.mark.parametrize( + "human_readable, expected_raw", + [ + ("12345", 12345), + ("5656 B", 5656), + ("89 bytes", 89), + ("1.0KiB", 1024), + ("3.2 GiB", 3435973836), + ], +) +def test_human_readable_to_raw(human_readable, expected_raw): + """Test the conversion to raw number of bytes.""" + assert _human_readable_to_raw(human_readable) == expected_raw + + +@pytest.mark.parametrize( + "workflow_status_response,expected_tests_result", + [ + ({"status": "finished"}, AnalysisTestStatus.passed), + ({"status": "failed"}, AnalysisTestStatus.failed), + ({"status": "running"}, AnalysisTestStatus.skipped), + ], +) +def test_workflow_execution_completes( + workflow_status_response, expected_tests_result, mock_data_fetcher +): + """Test the step definitions relative to the workflow execution completion. + + The tests should be skipped if the workflow is not finished, but should be run + otherwise. + """ + feature_file_path = os.path.join( + current_dir, + "gherkin_test_files", + "features", + "workflow-execution-completes.feature", + ) + mock_data_fetcher.get_workflow_status.return_value = workflow_status_response + _, test_results = parse_and_run_tests( + feature_file_path, + "test-workflow", + mock_data_fetcher, + ) + for scenario in test_results: + assert scenario.result == expected_tests_result + + +def test_log_content(mock_data_fetcher): + """Test the step definitions relative to the log content.""" + + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "log-content.feature" + ) + _, test_results = parse_and_run_tests( + feature_file_path, "test-workflow", mock_data_fetcher + ) + for scenario in test_results: + assert scenario.result in ( + AnalysisTestStatus.passed, + AnalysisTestStatus.skipped, + ) + + +def test_workflow_duration(mock_data_fetcher): + """Test the step definitions relative to the workflow duration.""" + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "workflow-duration.feature" + ) + _, test_results = parse_and_run_tests( + feature_file_path, + "test-workflow", + mock_data_fetcher, + ) + for scenario in test_results: + assert scenario.result in ( + AnalysisTestStatus.passed, + AnalysisTestStatus.skipped, + ) + + +def test_workspace_content(mock_data_fetcher): + """Test the step definitions relative to the workspace content.""" + + def get_mocked_workflow_disk_usage(workflow, parameters): + if parameters.get("summarize", False): + return { + "disk_usage_info": [ + {"name": "", "size": {"human_readable": "24 MiB", "raw": 25344320}}, + ] + } + else: + return mock_data_fetcher.get_workflow_disk_usage.return_value + + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "workspace-content.feature" + ) + + mock_data_fetcher.get_workflow_disk_usage.side_effect = ( + get_mocked_workflow_disk_usage + ) + _, test_results = parse_and_run_tests( + feature_file_path, "test-workflow", mock_data_fetcher + ) + + for scenario in test_results: + assert scenario.result in ( + AnalysisTestStatus.passed, + AnalysisTestStatus.skipped, + ) diff --git a/tests/test_gherkin_parser.py b/tests/test_gherkin_parser.py new file mode 100644 index 00000000..7d1d9f1d --- /dev/null +++ b/tests/test_gherkin_parser.py @@ -0,0 +1,168 @@ +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +from unittest.mock import patch +import pytest +import os +from reana_commons.gherkin_parser.parser import ( + parse_feature_file, + get_steps_list, + map_steps_to_functions, + run_tests, + parse_and_run_tests, + AnalysisTestStatus, +) +from reana_commons.gherkin_parser.errors import StepDefinitionNotFound + +current_dir = os.path.dirname(os.path.abspath(__file__)) + + +def test_parse_feature_file_okay(): + """Test for the parse_feature_file method.""" + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "test-gherkin-syntax.feature" + ) + + feature_name, feature = parse_feature_file(feature_file_path) + assert feature_name == "Test Feature" + assert len(feature) == 2 + assert feature[0]["name"] == "scenario 1" + assert len(feature[0]["steps"]) == 4 + assert feature[1]["name"] == "scenario 2" + assert len(feature[1]["steps"]) == 2 + + +def test_parse_feature_file_non_existing(): + """Test parsing a feature file that does not exist.""" + with pytest.raises(FileNotFoundError): + parse_feature_file("non-existing-feature.feature") + + +def test_get_steps_list(): + """Test for the get_steps_list method.""" + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "test-gherkin-syntax.feature" + ) + _, feature = parse_feature_file(feature_file_path) + steps = get_steps_list(feature) + assert len(steps) == 6 + assert steps[0] == ("Context", "this is a context clause") + assert steps[1] == ("Action", "this is an action clause") + assert steps[2] == ("Outcome", "this is an outcome clause") + assert steps[3] == ("Outcome", "this is another outcome clause") + assert steps[4] == ("Action", "the workflow is finished") + assert steps[5] == ("Outcome", "this should be tested") + + +@patch("reana_commons.gherkin_parser.data_fetcher.DataFetcherBase") +def test_map_steps_to_functions(mock_data_fetcher): + """Test for the map_steps_to_functions method.""" + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "log-content.feature" + ) + _, feature = parse_feature_file(feature_file_path) + steps = get_steps_list(feature) + step_mapping = map_steps_to_functions(steps, mock_data_fetcher) + assert len(step_mapping["Context"]) == 0 + assert len(step_mapping["Action"]) == 1 + assert step_mapping["Action"].keys() == {"the workflow is finished"} + assert len(step_mapping["Outcome"]) == 4 + + +def test_run_tests(mock_data_fetcher): + """Test for the run_tests method.""" + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "workflow-duration.feature" + ) + feature_name, feature = parse_feature_file(feature_file_path) + steps = get_steps_list(feature) + step_mapping = map_steps_to_functions(steps, mock_data_fetcher) + test_results = run_tests( + workflow="test_wf", + feature_name=feature_name, + feature_file=feature, + step_mapping=step_mapping, + ) + # Assert that each of the test results has a status of "passed" + for test_result in test_results: + assert test_result.result == AnalysisTestStatus.passed + + +def test_run_tests_no_feature_file(): + """Test for the parse_and_run_tests method when the feature file doesn't exist.""" + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "non-existing.feature" + ) + with pytest.raises(FileNotFoundError): + _, test_results = parse_and_run_tests( + feature_file_path=feature_file_path, + workflow="test_wf", + data_fetcher=None, + ) + + +def test_step_definition_not_found(): + """Test what happens when a step definition is not found.""" + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "test-gherkin-syntax.feature" + ) + _, feature = parse_feature_file(feature_file_path) + steps = get_steps_list(feature) + # Assert that the step mapping throws a StepDefinitionNotFound exception, since + # the steps are not defined + with pytest.raises(StepDefinitionNotFound): + map_steps_to_functions(steps, None) + + +def test_test_result_fail(mock_data_fetcher): + """Test that a failing test is detected.""" + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "failing-test.feature" + ) + + _, test_results = parse_and_run_tests( + feature_file_path, + "test-workflow", + mock_data_fetcher, + ) + assert test_results[0].result == AnalysisTestStatus.passed + assert test_results[1].result == AnalysisTestStatus.failed + assert test_results[2].result == AnalysisTestStatus.passed + + +@pytest.mark.parametrize( + "workflow_status_response,expected_tests_result,expected_error_log", + [ + ( + {"status": "finished"}, + AnalysisTestStatus.failed, + 'The workflow "test-workflow" is not "failed". Its status is "finished".', + ), + ({"status": "failed"}, AnalysisTestStatus.passed, None), + ], +) +def test_test_expected_workflow_fail_not_skipped( + workflow_status_response, + expected_tests_result, + expected_error_log, + mock_data_fetcher, +): + """Test what happens with expected failures. + + When the workflow status is "finished", the test should fail. + When the workflow status is "failed", the test should pass. + """ + feature_file_path = os.path.join( + current_dir, "gherkin_test_files", "features", "expected-failure.feature" + ) + mock_data_fetcher.get_workflow_status.return_value = workflow_status_response + _, test_results = parse_and_run_tests( + feature_file_path, + "test-workflow", + mock_data_fetcher, + ) + for scenario in test_results: + assert scenario.result == expected_tests_result + assert scenario.error_log == expected_error_log