From fb1ad0b2988c6df815e0a5642633c8b955dff083 Mon Sep 17 00:00:00 2001 From: Alex Flom Date: Tue, 3 Oct 2023 12:57:45 -0600 Subject: [PATCH] feat(transformer): Add CSV to YAML with empty writer (#48) Bootstraps component definitions with a rules view and a template. Also adds class for CSV to YAML transformations. Related PSCE-238 Majority of code generated by chatGPT Signed-off-by: Alex Flom --- poetry.lock | 12 +- .../transformers/test_csv_to_yaml.py | 88 +++++++++++ trestlebot/const.py | 2 + trestlebot/tasks/authored/compdef.py | 14 ++ trestlebot/transformers/csv_to_yaml.py | 137 ++++++++++++++++++ 5 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 tests/trestlebot/transformers/test_csv_to_yaml.py create mode 100644 trestlebot/transformers/csv_to_yaml.py diff --git a/poetry.lock b/poetry.lock index bc36ac8c..32602522 100644 --- a/poetry.lock +++ b/poetry.lock @@ -430,7 +430,7 @@ files = [ [[package]] name = "compliance-trestle" -version = "2.3.1.dev29+g725f6980" +version = "0.1.dev1134+g27533b4" description = "Tools to manage & autogenerate python objects representing the OSCAL layers/models" optional = false python-versions = "*" @@ -1830,6 +1830,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1837,8 +1838,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1855,6 +1863,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1862,6 +1871,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, diff --git a/tests/trestlebot/transformers/test_csv_to_yaml.py b/tests/trestlebot/transformers/test_csv_to_yaml.py new file mode 100644 index 00000000..290d295a --- /dev/null +++ b/tests/trestlebot/transformers/test_csv_to_yaml.py @@ -0,0 +1,88 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import csv +import pathlib +from dataclasses import fields + +import pytest +import ruamel.yaml as yaml + +from trestlebot.transformers.csv_to_yaml import YAMLBuilder +from trestlebot.transformers.trestle_rule import TrestleRule + + +@pytest.fixture(scope="function") +def setup_yaml_builder() -> YAMLBuilder: + return YAMLBuilder() + + +def write_sample_csv(csv_file: pathlib.Path) -> None: + with open(csv_file, "w", newline="") as csvfile: + fieldnames = [ + "RULE_ID", + "RULE_DESCRIPTION", + "PROFILE_DESCRIPTION", + "PROFILE_SOURCE", + "CONTROL_ID_LIST", + "COMPONENT_TITLE", + "COMPONENT_DESCRIPTION", + "COMPONENT_TYPE", + ] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + writer.writerow( + { + "RULE_ID": "Rule1", + "RULE_DESCRIPTION": "Description1", + "PROFILE_DESCRIPTION": "ProfileDesc1", + "PROFILE_SOURCE": "http://example.com", + "CONTROL_ID_LIST": "C1, C2", + "COMPONENT_TITLE": "Component1", + "COMPONENT_DESCRIPTION": "ComponentDesc1", + "COMPONENT_TYPE": "Type1", + } + ) + + +def test_read_from_csv(setup_yaml_builder: YAMLBuilder, tmp_trestle_dir: str) -> None: + csv_file = pathlib.Path(tmp_trestle_dir) / "test.csv" + write_sample_csv(csv_file) + setup_yaml_builder.read_from_csv(csv_file) + assert len(setup_yaml_builder._rules) == 1 + + +def test_write_to_yaml(setup_yaml_builder: YAMLBuilder, tmp_trestle_dir: str) -> None: + csv_file = pathlib.Path(tmp_trestle_dir) / "test.csv" + yaml_file = pathlib.Path(tmp_trestle_dir) / "test.yaml" + write_sample_csv(csv_file) + setup_yaml_builder.read_from_csv(csv_file) + setup_yaml_builder.write_to_yaml(yaml_file) + with open(yaml_file, "r") as f: + data = yaml.safe_load(f) + assert len(data) == 1 + + +def test_write_empty_trestle_rule_keys( + setup_yaml_builder: YAMLBuilder, tmp_trestle_dir: str +) -> None: + yaml_file = pathlib.Path(tmp_trestle_dir) / "test.yaml" + setup_yaml_builder.write_empty_trestle_rule_keys(yaml_file) + with open(yaml_file, "r") as f: + data = yaml.safe_load(f) + assert all(value == "" for value in data.values()) + expected_keys = {field.name for field in fields(TrestleRule)} + assert expected_keys == set(data.keys()) diff --git a/trestlebot/const.py b/trestlebot/const.py index b50db60f..5c6d2a37 100644 --- a/trestlebot/const.py +++ b/trestlebot/const.py @@ -45,3 +45,5 @@ COMPONENT_INFO_TAG = trestle_const.TRESTLE_TAG + "component-info" YAML_EXTENSION = ".yaml" + +RULES_VIEW_DIR = "rules" diff --git a/trestlebot/tasks/authored/compdef.py b/trestlebot/tasks/authored/compdef.py index 7ba4c86d..81935b9f 100644 --- a/trestlebot/tasks/authored/compdef.py +++ b/trestlebot/tasks/authored/compdef.py @@ -33,10 +33,12 @@ from trestle.core.profile_resolver import ProfileResolver from trestle.core.repository import AgileAuthoring +from trestlebot.const import RULES_VIEW_DIR from trestlebot.tasks.authored.base_authored import ( AuthoredObjectException, AuthorObjectBase, ) +from trestlebot.transformers.csv_to_yaml import YAMLBuilder class AuthoredComponentsDefinition(AuthorObjectBase): @@ -170,6 +172,18 @@ def create_new_default( cd_path.parent.mkdir(parents=True, exist_ok=True) comp_data.oscal_write(path=cd_path) # type: ignore + for component in comp_data.components: + ruledir: pathlib.Path = trestle_root.joinpath( + RULES_VIEW_DIR, + compdef_name, + component.title, + "rule_template.yaml", + ) + ruledir.parent.mkdir(parents=True, exist_ok=True) + + empty_yaml = YAMLBuilder() + empty_yaml.write_empty_trestle_rule_keys(ruledir) + def get_control_implementation( component: comp.DefinedComponent, source: str, description: str, controls: List[str] diff --git a/trestlebot/transformers/csv_to_yaml.py b/trestlebot/transformers/csv_to_yaml.py new file mode 100644 index 00000000..025b4f40 --- /dev/null +++ b/trestlebot/transformers/csv_to_yaml.py @@ -0,0 +1,137 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import csv +import json +import pathlib +from dataclasses import asdict, fields +from typing import Dict, List, Optional + +import ruamel.yaml as yaml +import trestle.tasks.csv_to_oscal_cd as csv_to_oscal_cd + +from trestlebot import const +from trestlebot.transformers.trestle_rule import ( + ComponentInfo, + Control, + Parameter, + Profile, + TrestleRule, +) + + +class YAMLBuilder: + def __init__(self) -> None: + """Initialize.""" + self._rules: List[TrestleRule] = [] + + def read_from_csv(self, filepath: pathlib.Path) -> None: + """Read from a CSV file and populate self._rules.""" + try: + with open(filepath, mode="r", newline="") as csv_file: + reader = csv.DictReader(csv_file) + for row in reader: + self._rules.append(self._csv_to_rule(row)) + except Exception as e: + raise CSVReadError(f"Failed to read from CSV file: {e}") + + def _csv_to_rule(self, row: Dict[str, str]) -> TrestleRule: + """Transform a CSV row to a TrestleRule object.""" + rule_info = self._extract_rule_info(row) + profile = self._extract_profile(row) + component_info = self._extract_component_info(row) + parameter = self._extract_parameter(row) + + return TrestleRule( + name=rule_info[const.NAME], + description=rule_info[const.DESCRIPTION], + component=component_info, + parameter=parameter, + profile=profile, + ) + + def _extract_rule_info(self, row: Dict[str, str]) -> Dict[str, str]: + """Extract rule information from a CSV row.""" + return { + "name": row.get(csv_to_oscal_cd.RULE_ID, ""), + "description": row.get(csv_to_oscal_cd.RULE_DESCRIPTION, ""), + } + + def _extract_profile(self, row: Dict[str, str]) -> Profile: + """Extract profile information from a CSV row.""" + controls_list = row.get(csv_to_oscal_cd.CONTROL_ID_LIST, "").split(", ") + return Profile( + description=row.get(csv_to_oscal_cd.PROFILE_DESCRIPTION, ""), + href=row.get(csv_to_oscal_cd.PROFILE_SOURCE, ""), + include_controls=[ + Control(id=control_id.strip()) for control_id in controls_list + ], + ) + + def _extract_parameter(self, row: Dict[str, str]) -> Optional[Parameter]: + """Extract parameter information from a CSV row.""" + parameter_name = row.get(csv_to_oscal_cd.PARAMETER_ID, None) + if parameter_name: + return Parameter( + name=parameter_name, + description=row.get(csv_to_oscal_cd.PARAMETER_DESCRIPTION, ""), + alternative_values=json.loads( + row.get(csv_to_oscal_cd.PARAMETER_VALUE_ALTERNATIVES, "{}") + ), + default_value=row.get(csv_to_oscal_cd.PARAMETER_VALUE_DEFAULT, ""), + ) + return None + + def _extract_component_info(self, row: Dict[str, str]) -> ComponentInfo: + """Extract component information from a CSV row.""" + return ComponentInfo( + name=row.get(csv_to_oscal_cd.COMPONENT_TITLE, ""), + type=row.get(csv_to_oscal_cd.COMPONENT_TYPE, ""), + description=row.get(csv_to_oscal_cd.COMPONENT_DESCRIPTION, ""), + ) + + def write_to_yaml(self, filepath: pathlib.Path) -> None: + """Write the rules to a YAML file.""" + try: + with open(filepath, "w") as yaml_file: + yaml.dump( + [asdict(rule) for rule in self._rules], yaml_file + ) # Use Python's built-in asdict + except Exception as e: + raise YAMLWriteError(f"Failed to write rules to YAML file: {e}") + + def write_empty_trestle_rule_keys(self, filepath: pathlib.Path) -> None: + """Write empty TrestleRule keys to a YAML file.""" + try: + empty_dict = {f.name: "" for f in fields(TrestleRule)} + with open(filepath, "w") as yaml_file: + yaml.dump(empty_dict, yaml_file) + except Exception as e: + raise YAMLWriteError( + f"Failed to write empty TrestleRule keys to YAML file: {e}" + ) + + +class YAMLWriteError(Exception): + """Exception raised for errors during YAML writing.""" + + pass + + +class CSVReadError(Exception): + """Exception raised for errors during CSV reading.""" + + pass