From 0492ecd47954410fa820e18d210ccb707deab1cc Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Wed, 29 Nov 2023 18:04:34 -0500 Subject: [PATCH] test: adds end to end tests for SSP creation and autosync Signed-off-by: Jennifer Power --- tests/conftest.py | 40 +++- tests/e2e/README.md | 2 + .../e2e/{test_e2e.py => test_e2e_compdef.py} | 146 +-------------- tests/e2e/test_e2e_ssp.py | 172 ++++++++++++++++++ tests/testutils.py | 88 +++++++++ tests/trestlebot/tasks/authored/test_ssp.py | 2 +- .../test_rules_transform_workflow.py | 2 +- trestlebot/tasks/authored/ssp.py | 14 +- 8 files changed, 325 insertions(+), 141 deletions(-) rename tests/e2e/{test_e2e.py => test_e2e_compdef.py} (67%) create mode 100644 tests/e2e/test_e2e_ssp.py diff --git a/tests/conftest.py b/tests/conftest.py index 5f391cb2..5b75ef26 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,6 +20,7 @@ import logging import os import pathlib +import subprocess from tempfile import TemporaryDirectory from typing import Any, Dict, Generator, Tuple, TypeVar @@ -28,7 +29,14 @@ from trestle.common.err import TrestleError from trestle.core.commands.init import InitCmd -from tests.testutils import clean +from tests.testutils import ( + E2E_BUILD_CONTEXT, + MOCK_SERVER_IMAGE_NAME, + TRESTLEBOT_TEST_IMAGE_NAME, + build_mock_server_image, + build_trestlebot_image, + clean, +) from trestlebot import const from trestlebot.transformers.trestle_rule import ( ComponentInfo, @@ -185,3 +193,33 @@ def test_rule() -> TrestleRule: ), ) return test_trestle_rule + + +# E2E test fixtures + + +@pytest.fixture(scope="package") +def podman_setup() -> YieldFixture[int]: + """Build the trestlebot container image and run the mock server in a pod.""" + + cleanup_trestlebot_image = build_trestlebot_image() + cleanup_mock_server_image = build_mock_server_image() + + # Create a pod + response = subprocess.run( + ["podman", "play", "kube", f"{E2E_BUILD_CONTEXT}/play-kube.yml"], check=True + ) + yield response.returncode + + # Clean up the container image, pod and mock server + try: + subprocess.run( + ["podman", "play", "kube", "--down", f"{E2E_BUILD_CONTEXT}/play-kube.yml"], + check=True, + ) + if cleanup_trestlebot_image: + subprocess.run(["podman", "rmi", TRESTLEBOT_TEST_IMAGE_NAME], check=True) + if cleanup_mock_server_image: + subprocess.run(["podman", "rmi", MOCK_SERVER_IMAGE_NAME], check=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to clean up podman resources: {e}") diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 28c43209..7dbcd418 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -51,6 +51,8 @@ To run the end-to-end tests, follow these steps: podman build -t localhost/trestlebot:latest -f Dockerfile . ``` +- When created tests that push to a branch, ensure the name is "test". This is because the mock API server is configured to only allow pushes to a branch named "test". + ## Future Improvements - Provide an option to use pre-built trestle-bot container images from a registry instead of building them locally. - Create endpoints that mock GitHub and GitLab API calls for pull request creation. diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e_compdef.py similarity index 67% rename from tests/e2e/test_e2e.py rename to tests/e2e/test_e2e_compdef.py index 384a966a..eb03964b 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e_compdef.py @@ -14,13 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. -"""E2E tests for the rules transform command.""" +"""E2E tests for commands for component definition authoring.""" import argparse import logging import pathlib import subprocess -from typing import Dict, List, Tuple +from typing import Dict, Tuple import pytest from git.repo import Repo @@ -30,9 +30,8 @@ from trestle.oscal.component import ComponentDefinition from trestle.oscal.profile import Profile -from tests.conftest import YieldFixture from tests.testutils import ( - args_dict_to_list, + build_test_command, load_from_json, setup_for_profile, setup_rules_view, @@ -48,136 +47,9 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -image_name = "localhost/trestlebot:latest" -mock_server_image_name = "localhost/mock-server:latest" -pod_name = "trestlebot-e2e-pod" -e2e_context = "tests/e2e" -container_file = "Dockerfile" - test_prof = "simplified_nist_profile" test_filter_prof = "simplified_filter_profile" -test_comp_name = "test-comp" - - -def image_exists(image_name: str) -> bool: - """Check if the image already exists.""" - try: - subprocess.check_output(["podman", "image", "inspect", image_name]) - return True - except subprocess.CalledProcessError: - return False - - -def build_transform_command(data_path: str, command_args: Dict[str, str]) -> List[str]: - """Build a command to be run in the shell for rules transform.""" - return [ - "podman", - "run", - "--pod", - pod_name, - "--entrypoint", - "trestlebot-rules-transform", - "--rm", - "-v", - f"{data_path}:/trestle", - "-w", - "/trestle", - image_name, - *args_dict_to_list(command_args), - ] - - -def build_create_cd_command(data_path: str, command_args: Dict[str, str]) -> List[str]: - """Build a command to be run in the shell for create cd.""" - return [ - "podman", - "run", - "--pod", - pod_name, - "--entrypoint", - "trestlebot-create-cd", - "--rm", - "-v", - f"{data_path}:/trestle", - "-w", - "/trestle", - image_name, - *args_dict_to_list(command_args), - ] - - -def build_trestlebot_image() -> bool: - """ - Build the trestlebot image. - - Returns: - Returns true if the image was built, false if it already exists. - """ - if not image_exists(image_name): - subprocess.run( - [ - "podman", - "build", - "-f", - container_file, - "-t", - image_name, - ], - check=True, - ) - return True - return False - - -def build_mock_server_image() -> bool: - """ - Build the mock server image. - - Returns: - Returns true if the image was built, false if it already exists. - """ - if not image_exists(mock_server_image_name): - subprocess.run( - [ - "podman", - "build", - "-f", - f"{e2e_context}/{container_file}", - "-t", - mock_server_image_name, - e2e_context, - ], - check=True, - ) - return True - return False - - -@pytest.fixture(scope="module") -def podman_setup() -> YieldFixture[int]: - """Build the trestlebot container image and run the mock server in a pod.""" - - cleanup_trestlebot_image = build_trestlebot_image() - cleanup_mock_server_image = build_mock_server_image() - - # Create a pod - response = subprocess.run( - ["podman", "play", "kube", f"{e2e_context}/play-kube.yml"], check=True - ) - yield response.returncode - - # Clean up the container image, pod and mock server - try: - subprocess.run( - ["podman", "play", "kube", "--down", f"{e2e_context}/play-kube.yml"], - check=True, - ) - if cleanup_trestlebot_image: - subprocess.run(["podman", "rmi", image_name], check=True) - if cleanup_mock_server_image: - subprocess.run(["podman", "rmi", mock_server_image_name], check=True) - except subprocess.CalledProcessError as e: - raise RuntimeError(f"Failed to clean up podman resources: {e}") +test_comp_name = "test_comp" @pytest.mark.slow @@ -250,20 +122,22 @@ def test_rules_transform_e2e( remote_url = "http://localhost:8080/test.git" repo.create_remote("origin", url=remote_url) - command = build_transform_command(tmp_repo_str, command_args) + command = build_test_command(tmp_repo_str, "rules-transform", command_args) run_response = subprocess.run(command, capture_output=True) assert run_response.returncode == response # Check that the component definition was created if response == SUCCESS_EXIT_CODE: if "skip-items" in command_args: - assert "input: test-comp.csv" not in run_response.stdout.decode("utf-8") + assert f"input: {test_comp_name}.csv" not in run_response.stdout.decode( + "utf-8" + ) else: comp_path: pathlib.Path = ModelUtils.get_model_path_for_name_and_class( tmp_repo_path, test_comp_name, ComponentDefinition, FileContentType.JSON ) assert comp_path.exists() - assert "input: test-comp.csv" in run_response.stdout.decode("utf-8") + assert f"input: {test_comp_name}.csv" in run_response.stdout.decode("utf-8") branch = command_args["branch"] assert ( f"Changes pushed to {branch} successfully." @@ -384,7 +258,7 @@ def test_create_cd_e2e( remote_url = "http://localhost:8080/test.git" repo.create_remote("origin", url=remote_url) - command = build_create_cd_command(tmp_repo_str, command_args) + command = build_test_command(tmp_repo_str, "create-cd", command_args) run_response = subprocess.run(command, cwd=tmp_repo_path, capture_output=True) assert run_response.returncode == response diff --git a/tests/e2e/test_e2e_ssp.py b/tests/e2e/test_e2e_ssp.py new file mode 100644 index 00000000..2bbf0148 --- /dev/null +++ b/tests/e2e/test_e2e_ssp.py @@ -0,0 +1,172 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +E2E tests for SSP creation and autosync workflow. + +Notes that this should be the only E2E for auto-syncing since the UX is the same for each model. +Any model specific test should be under workflows. +""" + +import argparse +import logging +import os +import pathlib +import subprocess +from typing import Dict, Tuple + +import pytest +from git.repo import Repo +from trestle.common.model_utils import ModelUtils +from trestle.core.commands.author.ssp import SSPGenerate +from trestle.core.commands.init import InitCmd +from trestle.core.models.file_content_type import FileContentType +from trestle.oscal.ssp import SystemSecurityPlan + +from tests.testutils import build_test_command, setup_for_ssp +from trestlebot.const import ERROR_EXIT_CODE, INVALID_ARGS_EXIT_CODE, SUCCESS_EXIT_CODE +from trestlebot.tasks.authored.ssp import AuthoredSSP, SSPIndex + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +test_prof = "simplified_nist_profile" +test_comp_name = "test_comp" +test_ssp_md = "md_ssp" +test_ssp_name = "test_ssp" + + +@pytest.mark.slow +@pytest.mark.parametrize( + "test_name, command_args, response, skip_create", + [ + ( + "success/happy path", + { + "branch": "test", + "markdown-path": test_ssp_md, + "oscal-model": "ssp", + "committer-name": "test", + "committer-email": "test@email.com", + "ssp-index": "ssp-index.json", + }, + SUCCESS_EXIT_CODE, + False, + ), + ( + "failure/missing-ssp-index", + { + "branch": "test", + "markdown-path": test_ssp_md, + "oscal-model": "ssp", + "committer-name": "test", + "committer-email": "test@email.com", + }, + ERROR_EXIT_CODE, + True, + ), + ( + "failure/missing args", + { + "branch": "test", + "oscal-model": "ssp", + "committer-name": "test", + "committer-email": "test@email.com", + }, + INVALID_ARGS_EXIT_CODE, + False, + ), + ], +) +def test_ssp_editing_e2e( + tmp_repo: Tuple[str, Repo], + podman_setup: int, + test_name: str, + command_args: Dict[str, str], + response: int, + skip_create: bool, +) -> None: + """Test the trestlebot autosync command with SSPs.""" + # Check that the container image was built successfully + # and the mock server is running + assert podman_setup == 0 + + logger.info(f"Running test: {test_name}") + + tmp_repo_str, repo = tmp_repo + + tmp_repo_path = pathlib.Path(tmp_repo_str) + + # Create a trestle workspace in the temporary git repository + args = argparse.Namespace( + verbose=0, + trestle_root=tmp_repo_path, + full=True, + local=False, + govdocs=False, + ) + init = InitCmd() + init._run(args) + + args = setup_for_ssp(tmp_repo_path, test_prof, [test_comp_name], test_ssp_md) + + # Create or generate the SSP + + if not skip_create: + index_path = os.path.join(tmp_repo_str, "ssp-index.json") + ssp_index = SSPIndex(index_path) + authored_ssp = AuthoredSSP(tmp_repo_str, ssp_index) + authored_ssp.create_new_default( + test_ssp_name, + test_prof, + [test_comp_name], + test_ssp_md, + ) + else: + ssp_generate = SSPGenerate() + assert ssp_generate._run(args) == 0 + + ssp_path: pathlib.Path = ModelUtils.get_model_path_for_name_and_class( + tmp_repo_path, + test_ssp_name, + SystemSecurityPlan, + FileContentType.JSON, + ) + assert not ssp_path.exists() + + remote_url = "http://localhost:8080/test.git" + repo.create_remote("origin", url=remote_url) + + command = build_test_command(tmp_repo_str, "autosync", command_args) + run_response = subprocess.run(command, capture_output=True) + assert run_response.returncode == response + + # Check that the ssp was pushed to the remote + if response == SUCCESS_EXIT_CODE: + branch = command_args["branch"] + assert ( + f"Changes pushed to {branch} successfully." + in run_response.stdout.decode("utf-8") + ) + + # Check that the correct files are present with the correct content + assert (tmp_repo_path / command_args["markdown-path"]).exists() + ssp_index.reload() + assert ssp_index.get_profile_by_ssp(test_ssp_name) == test_prof + assert ssp_index.get_comps_by_ssp(test_ssp_name) == [test_comp_name] + assert ssp_index.get_leveraged_by_ssp(test_ssp_name) is None + assert ssp_path.exists() diff --git a/tests/testutils.py b/tests/testutils.py index 0e0c8b28..3c5d7f23 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -20,6 +20,7 @@ import json import pathlib import shutil +import subprocess from typing import Dict, List, Optional from git.repo import Repo @@ -41,6 +42,13 @@ JSON_TEST_DATA_PATH = pathlib.Path("tests/data/json/").resolve() YAML_TEST_DATA_PATH = pathlib.Path("tests/data/yaml/").resolve() +# E2E test constants +TRESTLEBOT_TEST_IMAGE_NAME = "localhost/trestlebot:latest" +MOCK_SERVER_IMAGE_NAME = "localhost/mock-server:latest" +TRESTLEBOT_TEST_POD_NAME = "trestlebot-e2e-pod" +E2E_BUILD_CONTEXT = "tests/e2e" +CONTAINER_FILE_NAME = "Dockerfile" + def clean(repo_path: str, repo: Optional[Repo]) -> None: """Clean up the temporary Git repository.""" @@ -269,3 +277,83 @@ def replace_string_in_file(file_path: str, old_string: str, new_string: str) -> # Write the updated content back to the file with open(file_path, "w") as file: file.write(updated_content) + + +# E2E test utils + + +def _image_exists(image_name: str) -> bool: + """Check if the image already exists.""" + try: + subprocess.check_output(["podman", "image", "inspect", image_name]) + return True + except subprocess.CalledProcessError: + return False + + +def build_trestlebot_image() -> bool: + """ + Build the trestlebot image. + + Returns: + Returns true if the image was built, false if it already exists. + """ + if not _image_exists(TRESTLEBOT_TEST_IMAGE_NAME): + subprocess.run( + [ + "podman", + "build", + "-f", + CONTAINER_FILE_NAME, + "-t", + TRESTLEBOT_TEST_IMAGE_NAME, + ], + check=True, + ) + return True + return False + + +def build_mock_server_image() -> bool: + """ + Build the mock server image. + + Returns: + Returns true if the image was built, false if it already exists. + """ + if not _image_exists(MOCK_SERVER_IMAGE_NAME): + subprocess.run( + [ + "podman", + "build", + "-f", + f"{E2E_BUILD_CONTEXT}/{CONTAINER_FILE_NAME}", + "-t", + MOCK_SERVER_IMAGE_NAME, + E2E_BUILD_CONTEXT, + ], + check=True, + ) + return True + return False + + +def build_test_command( + data_path: str, command_name: str, command_args: Dict[str, str] +) -> List[str]: + """Build a command to be run in the shell for trestlebot""" + return [ + "podman", + "run", + "--pod", + TRESTLEBOT_TEST_POD_NAME, + "--entrypoint", + f"trestlebot-{command_name}", + "--rm", + "-v", + f"{data_path}:/trestle", + "-w", + "/trestle", + TRESTLEBOT_TEST_IMAGE_NAME, + *args_dict_to_list(command_args), + ] diff --git a/tests/trestlebot/tasks/authored/test_ssp.py b/tests/trestlebot/tasks/authored/test_ssp.py index 2e871e1a..50107fba 100644 --- a/tests/trestlebot/tasks/authored/test_ssp.py +++ b/tests/trestlebot/tasks/authored/test_ssp.py @@ -206,7 +206,7 @@ def test_write_new_ssp_index(tmp_trestle_dir: str) -> None: ssp_index.write_out() # Reread the ssp index from JSON - ssp_index = SSPIndex(ssp_index_path) + ssp_index.reload() assert ssp_index.get_profile_by_ssp(test_ssp_output) == test_prof assert test_comp in ssp_index.get_comps_by_ssp(test_ssp_output) diff --git a/tests/workflows/test_rules_transform_workflow.py b/tests/workflows/test_rules_transform_workflow.py index 7fa83a54..9d3e8b33 100644 --- a/tests/workflows/test_rules_transform_workflow.py +++ b/tests/workflows/test_rules_transform_workflow.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Component test the rules transformation workflow.""" +"""Test the rules transformation workflow.""" import os import pathlib diff --git a/trestlebot/tasks/authored/ssp.py b/trestlebot/tasks/authored/ssp.py index c36de38b..ece051f2 100644 --- a/trestlebot/tasks/authored/ssp.py +++ b/trestlebot/tasks/authored/ssp.py @@ -51,11 +51,14 @@ def __init__(self, index_path: str) -> None: self.profile_by_ssp: Dict[str, str] = {} self.comps_by_ssp: Dict[str, List[str]] = {} self.leveraged_ssp_by_ssp: Dict[str, str] = {} + self._load() + def _load(self) -> None: + """Load the index from the index file""" # Try to load the current file. If it does not exist, # create an empty JSON file. try: - with open(index_path, "r") as file: + with open(self._index_path, "r") as file: json_data = json.load(file) for ssp_name, ssp_info in json_data.items(): @@ -77,9 +80,16 @@ def __init__(self, index_path: str) -> None: ] except FileNotFoundError: - with open(index_path, "w") as file: + with open(self._index_path, "w") as file: json.dump({}, file) + def reload(self) -> None: + """Reload the index from the index file""" + self.profile_by_ssp = {} + self.comps_by_ssp = {} + self.leveraged_ssp_by_ssp = {} + self._load() + def get_comps_by_ssp(self, ssp_name: str) -> List[str]: """Return list of compdefs associated with the SSP""" try: