From b466e9f8c5467197c2765ccab18991349611cf60 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Mon, 9 Oct 2023 15:56:10 -0400 Subject: [PATCH] PSCE-245 - Add pydantic for basic data validation of Trestle Rule (#50) * refactor: reworks csv_to_yaml for pydantic Update write empty keys to write default keys Create transformer for CSV reading and writing Moves CSV specific logic t csv_transformer Moves YAML specific logic to yaml_transformer and adds writing logic Signed-off-by: Jennifer Power * docs: updates comment Signed-off-by: Jennifer Power * docs: improves comment and variable naming Signed-off-by: Jennifer Power * refactor: reduce extra code in RulesYAMLTransformer for simplicity Signed-off-by: Jennifer Power * feat: adds Field aliases to remove underscores from YAML Signed-off-by: Jennifer Power * refactor: breaks transformer into ToRuleTransformer and FromRuleTransformer To align with upstream trestle, the transformers are broken down into single responsibilties and importing the Transformer Base. Signed-off-by: Jennifer Power --------- Signed-off-by: Jennifer Power --- poetry.lock | 161 +++++++--------- pyproject.toml | 1 + tests/conftest.py | 106 +++++++++- tests/data/yaml/test_invalid_rule.yaml | 17 ++ tests/data/yaml/test_rule_invalid_params.yaml | 17 ++ .../tasks/test_rule_transform_task.py | 10 +- .../transformers/test_csv_to_yaml.py | 34 ++-- .../transformers/test_csv_transformer.py | 70 +++++++ .../transformers/test_yaml_to_csv.py | 136 ------------- .../transformers/test_yaml_transformer.py | 102 ++++++++++ trestlebot/const.py | 7 +- trestlebot/tasks/authored/compdef.py | 2 +- trestlebot/tasks/rule_transform_task.py | 12 +- trestlebot/transformers/__init__.py | 4 + trestlebot/transformers/base_transformer.py | 44 +++++ trestlebot/transformers/csv_to_yaml.py | 111 ++++------- .../{yaml_to_csv.py => csv_transformer.py} | 181 ++++++++++-------- trestlebot/transformers/trestle_rule.py | 45 ++--- trestlebot/transformers/yaml_transformer.py | 136 +++++++++++++ 19 files changed, 759 insertions(+), 437 deletions(-) create mode 100644 tests/data/yaml/test_invalid_rule.yaml create mode 100644 tests/data/yaml/test_rule_invalid_params.yaml create mode 100644 tests/trestlebot/transformers/test_csv_transformer.py delete mode 100644 tests/trestlebot/transformers/test_yaml_to_csv.py create mode 100644 tests/trestlebot/transformers/test_yaml_transformer.py create mode 100644 trestlebot/transformers/base_transformer.py rename trestlebot/transformers/{yaml_to_csv.py => csv_transformer.py} (55%) create mode 100644 trestlebot/transformers/yaml_transformer.py diff --git a/poetry.lock b/poetry.lock index 32602522..cd7d2046 100644 --- a/poetry.lock +++ b/poetry.lock @@ -430,7 +430,7 @@ files = [ [[package]] name = "compliance-trestle" -version = "0.1.dev1134+g27533b4" +version = "2.3.1.dev29+g725f6980" description = "Tools to manage & autogenerate python objects representing the OSCAL layers/models" optional = false python-versions = "*" @@ -1518,52 +1518,52 @@ files = [ [[package]] name = "pydantic" -version = "1.10.13" +version = "1.10.2" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, - {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, - {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, - {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, - {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, - {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, - {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, - {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, - {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, - {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, + {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, + {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, + {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, + {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, + {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, + {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, + {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, + {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] [package.dependencies] email-validator = {version = ">=1.0.3", optional = true, markers = "extra == \"email\""} -typing-extensions = ">=4.2.0" +typing-extensions = ">=4.1.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -1830,7 +1830,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1838,15 +1837,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1863,7 +1855,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1871,7 +1862,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1965,48 +1955,41 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] [[package]] name = "ruamel-yaml-clib" -version = "0.2.7" +version = "0.2.8" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:370445fd795706fd291ab00c9df38a0caed0f17a6fb46b0f607668ecb16ce763"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win32.whl", hash = "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:40d030e2329ce5286d6b231b8726959ebbe0404c92f0a578c0e2482182e38282"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c3ca1fbba4ae962521e5eb66d72998b51f0f4d0f608d3c0347a48e1af262efa7"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3243f48ecd450eddadc2d11b5feb08aca941b5cd98c9b1db14b2fd128be8c697"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win32.whl", hash = "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf9a6bc4a0221538b1a7de3ed7bca4c93c02346853f44e1cd764be0023cd3640"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win32.whl", hash = "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5"}, - {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, + {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, ] [[package]] @@ -2177,4 +2160,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "b6d05c2abf33d5cb66d9adaaa077061fea617623672e5ba69148494073b2cfcb" +content-hash = "d2725065f43dfc637e8ce40a2e6c9b9587b1ec3b01a85353acfb9c60a4e4ff1d" diff --git a/pyproject.toml b/pyproject.toml index 76eba6ba..a28c9c94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ compliance-trestle = {git = "https://github.com/IBM/compliance-trestle.git", rev github3-py = "^4.0.1" python-gitlab = "^3.15.0" ruamel-yaml = "^0.17.32" +pydantic = "1.10.2" [tool.poetry.group.dev.dependencies] flake8 = "^6.0.0" diff --git a/tests/conftest.py b/tests/conftest.py index eea79001..8383feaf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,13 +20,22 @@ import os import pathlib from tempfile import TemporaryDirectory -from typing import Generator, Tuple, TypeVar +from typing import Any, Dict, Generator, Tuple, TypeVar import pytest from git.repo import Repo from trestle.common.err import TrestleError from trestle.core.commands.init import InitCmd +from trestlebot import const +from trestlebot.transformers.trestle_rule import ( + ComponentInfo, + Control, + Parameter, + Profile, + TrestleRule, +) + T = TypeVar("T") @@ -74,3 +83,98 @@ def tmp_trestle_dir() -> YieldFixture[str]: f"Initialization failed for temporary trestle directory: {e}." ) yield tmpdir + + +@pytest.fixture(scope="function") +def valid_rule_data() -> Dict[str, Any]: + return { + const.RULE_INFO_TAG: { + const.NAME: "example_rule_1", + const.DESCRIPTION: "My rule description for example rule 1", + const.PROFILE: { + const.DESCRIPTION: "Simple NIST Profile", + const.HREF: "profiles/simplified_nist_profile/profile.json", + }, + const.PARAMETER: { + const.NAME: "prm_1", + const.DESCRIPTION: "prm_1 description", + const.ALTERNATIVE_VALUES: { + "default": "5%", + "5pc": "5%", + "10pc": "10%", + "15pc": "15%", + "20pc": "20%", + }, + const.DEFAULT_VALUE: "5%", + }, + } + } + + +@pytest.fixture(scope="function") +def invalid_param_rule_data() -> Dict[str, Any]: + return { + const.RULE_INFO_TAG: { + const.NAME: "example_rule_1", + const.DESCRIPTION: "My rule description for example rule 1", + const.PROFILE: { + const.DESCRIPTION: "Simple NIST Profile", + const.HREF: "profiles/simplified_nist_profile/profile.json", + }, + const.PARAMETER: { + const.NAME: "prm_1", + const.DESCRIPTION: "prm_1 description", + const.ALTERNATIVE_VALUES: { + "5pc": "5%", + "10pc": "10%", + "15pc": "15%", + "20pc": "20%", + }, + const.DEFAULT_VALUE: "5%", + }, + } + } + + +@pytest.fixture(scope="function") +def missing_key_rule_data() -> Dict[str, Any]: + return { + const.RULE_INFO_TAG: { + const.DESCRIPTION: "My rule description for example rule 1", + const.PROFILE: { + const.DESCRIPTION: "Simple NIST Profile", + const.HREF: "profiles/simplified_nist_profile/profile.json", + }, + const.PARAMETER: { + const.NAME: "prm_1", + const.DESCRIPTION: "prm_1 description", + const.ALTERNATIVE_VALUES: { + "default": "5%", + "5pc": "5%", + "10pc": "10%", + "15pc": "15%", + "20pc": "20%", + }, + const.DEFAULT_VALUE: "5%", + }, + } + } + + +@pytest.fixture(scope="function") +def test_rule() -> TrestleRule: + test_trestle_rule: TrestleRule = TrestleRule( + name="test", + description="test", + component=ComponentInfo(name="test_comp", type="test", description="test"), + parameter=Parameter( + name="test", + description="test", + alternative_values={}, + default_value="test", + ), + profile=Profile( + description="test", href="test", include_controls=[Control(id="ac-1")] + ), + ) + return test_trestle_rule diff --git a/tests/data/yaml/test_invalid_rule.yaml b/tests/data/yaml/test_invalid_rule.yaml new file mode 100644 index 00000000..6062e47a --- /dev/null +++ b/tests/data/yaml/test_invalid_rule.yaml @@ -0,0 +1,17 @@ +x-trestle-rule-info: + name: example_rule_1 + description: My rule description for example rule 1 + parameter: + name: prm_1 + description: prm_1 description + alternative-values: "invalid" + default-value: true + profile: + description: Simple NIST Profile + href: profiles/simplified_nist_profile/profile.json + include-controls: + - id: ac-2 +x-trestle-component-info: + name: Component 1 + description: Component 1 description + type: service diff --git a/tests/data/yaml/test_rule_invalid_params.yaml b/tests/data/yaml/test_rule_invalid_params.yaml new file mode 100644 index 00000000..3b4acdcd --- /dev/null +++ b/tests/data/yaml/test_rule_invalid_params.yaml @@ -0,0 +1,17 @@ +x-trestle-rule-info: + name: example_rule_1 + description: My rule description for example rule 1 + parameter: + name: prm_1 + description: prm_1 description + alternative-values: {'5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} + default-value: '5%' + profile: + description: Simple NIST Profile + href: profiles/simplified_nist_profile/profile.json + include-controls: + - id: ac-2 +x-trestle-component-info: + name: Component 1 + description: Component 1 description + type: service diff --git a/tests/trestlebot/tasks/test_rule_transform_task.py b/tests/trestlebot/tasks/test_rule_transform_task.py index 59c75464..9fb61200 100644 --- a/tests/trestlebot/tasks/test_rule_transform_task.py +++ b/tests/trestlebot/tasks/test_rule_transform_task.py @@ -28,7 +28,7 @@ from tests.testutils import setup_rules_view from trestlebot.tasks.base_task import TaskException from trestlebot.tasks.rule_transform_task import RuleTransformTask -from trestlebot.transformers.yaml_to_csv import RulesYAMLTransformer +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer test_comp = "test_comp" @@ -39,7 +39,7 @@ def test_rule_transform_task(tmp_trestle_dir: str) -> None: """Test rule transform task.""" trestle_root = pathlib.Path(tmp_trestle_dir) setup_rules_view(trestle_root, test_comp, test_rules_dir) - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() rule_transform_task = RuleTransformTask( tmp_trestle_dir, test_rules_dir, transformer ) @@ -70,7 +70,7 @@ def test_rule_transform_task_with_no_rules(tmp_trestle_dir: str) -> None: """Test rule transform task with no rules.""" trestle_root = pathlib.Path(tmp_trestle_dir) setup_rules_view(trestle_root, test_comp, test_rules_dir, skip_rules=True) - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() rule_transform_task = RuleTransformTask( tmp_trestle_dir, test_rules_dir, transformer ) @@ -85,7 +85,7 @@ def test_rule_transform_task_with_invalid_rule(tmp_trestle_dir: str) -> None: """Test rule transform task with invalid rule.""" trestle_root = pathlib.Path(tmp_trestle_dir) setup_rules_view(trestle_root, test_comp, test_rules_dir, incomplete_rule=True) - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() rule_transform_task = RuleTransformTask( tmp_trestle_dir, test_rules_dir, transformer ) @@ -100,7 +100,7 @@ def test_rule_transform_task_with_skip(tmp_trestle_dir: str) -> None: """Test rule transform task with skip.""" trestle_root = pathlib.Path(tmp_trestle_dir) setup_rules_view(trestle_root, test_comp, test_rules_dir) - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() rule_transform_task = RuleTransformTask( tmp_trestle_dir, test_rules_dir, transformer, skip_model_list=[test_comp] ) diff --git a/tests/trestlebot/transformers/test_csv_to_yaml.py b/tests/trestlebot/transformers/test_csv_to_yaml.py index 290d295a..42a600a0 100644 --- a/tests/trestlebot/transformers/test_csv_to_yaml.py +++ b/tests/trestlebot/transformers/test_csv_to_yaml.py @@ -16,13 +16,12 @@ import csv import pathlib -from dataclasses import fields import pytest -import ruamel.yaml as yaml +from ruamel.yaml import YAML from trestlebot.transformers.csv_to_yaml import YAMLBuilder -from trestlebot.transformers.trestle_rule import TrestleRule +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer @pytest.fixture(scope="function") @@ -71,18 +70,29 @@ def test_write_to_yaml(setup_yaml_builder: YAMLBuilder, tmp_trestle_dir: str) -> write_sample_csv(csv_file) setup_yaml_builder.read_from_csv(csv_file) setup_yaml_builder.write_to_yaml(yaml_file) + yaml = YAML(typ="safe") with open(yaml_file, "r") as f: - data = yaml.safe_load(f) - assert len(data) == 1 + data = yaml.load(f) + # The file will contain a separate YAML document for each rule + assert len(data) == 2 -def test_write_empty_trestle_rule_keys( +def test_default_test_trestle_rule_keys( setup_yaml_builder: YAMLBuilder, tmp_trestle_dir: str ) -> None: yaml_file = pathlib.Path(tmp_trestle_dir) / "test.yaml" - setup_yaml_builder.write_empty_trestle_rule_keys(yaml_file) - with open(yaml_file, "r") as f: - data = yaml.safe_load(f) - assert all(value == "" for value in data.values()) - expected_keys = {field.name for field in fields(TrestleRule)} - assert expected_keys == set(data.keys()) + setup_yaml_builder.write_default_trestle_rule_keys(yaml_file) + + # Check that the YAML file written is valid and integrates with the rule + # YAML transformer + transformer = ToRulesYAMLTransformer() + rule = transformer.transform(yaml_file.read_text()) + + assert rule.name == "example rule" + assert rule.description == "example description" + assert rule.component.name == "example component" + assert rule.component.description == "example description" + assert rule.component.type == "service" + assert rule.profile.description == "example profile" + assert rule.profile.href == "example href" + assert len(rule.profile.include_controls) == 1 diff --git a/tests/trestlebot/transformers/test_csv_transformer.py b/tests/trestlebot/transformers/test_csv_transformer.py new file mode 100644 index 00000000..6dc61bb4 --- /dev/null +++ b/tests/trestlebot/transformers/test_csv_transformer.py @@ -0,0 +1,70 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Test for CSV Transformer.""" + +import csv +import pathlib +from typing import List + +import pytest + +from trestlebot.transformers.csv_transformer import CSVBuilder +from trestlebot.transformers.trestle_rule import TrestleRule + + +def test_csv_builder(test_rule: TrestleRule, tmp_trestle_dir: str) -> None: + """Test CSV builder on a happy path""" + + csv_builder = CSVBuilder() + csv_builder.add_row(test_rule) + + assert len(csv_builder._rows) == 1 + row = csv_builder._rows[0] + assert row["Rule_Id"] == test_rule.name + assert row["Rule_Description"] == test_rule.description + assert row["Component_Title"] == test_rule.component.name + assert row["Component_Type"] == test_rule.component.type + assert row["Component_Description"] == test_rule.component.description + assert row["Control_Id_List"] == "ac-1" + assert row["Parameter_Id"] == test_rule.parameter.name # type: ignore + assert row["Parameter_Description"] == test_rule.parameter.description # type: ignore + assert row["Parameter_Value_Alternatives"] == "{}" + assert row["Parameter_Value_Default"] == test_rule.parameter.default_value # type: ignore + assert row["Profile_Description"] == test_rule.profile.description + assert row["Profile_Source"] == test_rule.profile.href + + trestle_root = pathlib.Path(tmp_trestle_dir) + tmp_csv_path = trestle_root.joinpath("test.csv") + csv_builder.write_to_file(tmp_csv_path) + + assert tmp_csv_path.exists() + + first_row: List[str] = [] + with open(tmp_csv_path, "r", newline="") as csvfile: + csv_reader = csv.reader(csvfile) + first_row = next(csv_reader) + + for column in csv_builder._csv_columns.get_required_column_names(): + assert column in first_row + + +def test_validate_row() -> None: + """Test validate row with an invalid row.""" + row = {"Rule_Id": "test"} + csv_builder = CSVBuilder() + with pytest.raises(RuntimeError, match="Row missing key: *"): + csv_builder.validate_row(row) diff --git a/tests/trestlebot/transformers/test_yaml_to_csv.py b/tests/trestlebot/transformers/test_yaml_to_csv.py deleted file mode 100644 index ef7e0f6a..00000000 --- a/tests/trestlebot/transformers/test_yaml_to_csv.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/python - -# Copyright 2023 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Test for YAML to CSV Transformer.""" - -import csv -import pathlib -from typing import List - -import pytest - -from tests.testutils import YAML_TEST_DATA_PATH -from trestlebot.transformers.trestle_rule import ( - ComponentInfo, - Control, - Parameter, - Profile, - RulesTransformerException, - TrestleRule, -) -from trestlebot.transformers.yaml_to_csv import CSVBuilder, RulesYAMLTransformer - - -test_comp = "my comp" - - -def test_csv_builder(tmp_trestle_dir: str) -> None: - """Test CSV builder on a happy path""" - test_trestle_rule: TrestleRule = TrestleRule( - name="test", - description="test", - component=ComponentInfo(name=test_comp, type="test", description="test"), - parameter=Parameter( - name="test", description="test", alternative_values={}, default_value="test" - ), - profile=Profile( - description="test", href="test", include_controls=[Control(id="ac-1")] - ), - ) - csv_builder = CSVBuilder() - csv_builder.add_row(test_trestle_rule) - - assert len(csv_builder._rows) == 1 - row = csv_builder._rows[0] - assert row["Rule_Id"] == test_trestle_rule.name - assert row["Rule_Description"] == test_trestle_rule.description - assert row["Component_Title"] == test_trestle_rule.component.name - assert row["Component_Type"] == test_trestle_rule.component.type - assert row["Component_Description"] == test_trestle_rule.component.description - assert row["Control_Id_List"] == "ac-1" - assert row["Parameter_Id"] == test_trestle_rule.parameter.name # type: ignore - assert row["Parameter_Description"] == test_trestle_rule.parameter.description # type: ignore - assert row["Parameter_Value_Alternatives"] == "{}" - assert row["Parameter_Value_Default"] == test_trestle_rule.parameter.default_value # type: ignore - assert row["Profile_Description"] == test_trestle_rule.profile.description - assert row["Profile_Source"] == test_trestle_rule.profile.href - - trestle_root = pathlib.Path(tmp_trestle_dir) - tmp_csv_path = trestle_root.joinpath("test.csv") - csv_builder.write_to_file(tmp_csv_path) - - assert tmp_csv_path.exists() - - first_row: List[str] = [] - with open(tmp_csv_path, "r", newline="") as csvfile: - csv_reader = csv.reader(csvfile) - first_row = next(csv_reader) - - for column in csv_builder._csv_columns.get_required_column_names(): - assert column in first_row - - -def test_validate_row() -> None: - """Test validate row with an invalid row.""" - row = {"Rule_Id": "test"} - csv_builder = CSVBuilder() - with pytest.raises(RuntimeError, match="Row missing key: *"): - csv_builder.validate_row(row) - - -def test_rule_transformer() -> None: - """Test rule transformer.""" - # load rule from path and close the file - # get the file info as a string - rule_path = YAML_TEST_DATA_PATH / "test_complete_rule.yaml" - rule_file = open(rule_path, "r") - rule_file_info = rule_file.read() - rule_file.close() - - transformer = RulesYAMLTransformer() - rule = transformer.transform(rule_file_info) - - assert rule.name == "example_rule_1" - assert rule.description == "My rule description for example rule 1" - assert rule.component.name == "Component 1" - assert rule.component.type == "service" - assert rule.component.description == "Component 1 description" - assert rule.parameter is not None - assert rule.parameter.name == "prm_1" - assert rule.parameter.description == "prm_1 description" - assert rule.parameter.alternative_values == { - "default": "5%", - "5pc": "5%", - "10pc": "10%", - "15pc": "15%", - "20pc": "20%", - } - assert rule.parameter.default_value == "5%" - assert rule.profile.description == "Simple NIST Profile" - assert rule.profile.href == "profiles/simplified_nist_profile/profile.json" - - -def test_rules_transform_with_invalid_rule() -> None: - """Test rules transform with invalid rule.""" - # Generate test json string - test_string = '{"test_json": "test"}' - transformer = RulesYAMLTransformer() - - with pytest.raises( - RulesTransformerException, - match="Missing key in YAML file: 'x-trestle-rule-info'", - ): - transformer.transform(test_string) diff --git a/tests/trestlebot/transformers/test_yaml_transformer.py b/tests/trestlebot/transformers/test_yaml_transformer.py new file mode 100644 index 00000000..55bbb14d --- /dev/null +++ b/tests/trestlebot/transformers/test_yaml_transformer.py @@ -0,0 +1,102 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Test for YAML Transformer.""" + +import pytest + +from tests.testutils import YAML_TEST_DATA_PATH +from trestlebot.transformers.base_transformer import RulesTransformerException +from trestlebot.transformers.trestle_rule import TrestleRule +from trestlebot.transformers.yaml_transformer import ( + FromRulesYAMLTransformer, + ToRulesYAMLTransformer, +) + + +def test_rule_transformer() -> None: + """Test rule transformer.""" + # load rule from path and close the file + # get the file info as a string + rule_path = YAML_TEST_DATA_PATH / "test_complete_rule.yaml" + rule_file = open(rule_path, "r") + rule_file_info = rule_file.read() + rule_file.close() + + transformer = ToRulesYAMLTransformer() + rule = transformer.transform(rule_file_info) + + assert rule.name == "example_rule_1" + assert rule.description == "My rule description for example rule 1" + assert rule.component.name == "Component 1" + assert rule.component.type == "service" + assert rule.component.description == "Component 1 description" + assert rule.parameter is not None + assert rule.parameter.name == "prm_1" + assert rule.parameter.description == "prm_1 description" + assert rule.parameter.alternative_values == { + "default": "5%", + "5pc": "5%", + "10pc": "10%", + "15pc": "15%", + "20pc": "20%", + } + assert rule.parameter.default_value == "5%" + assert rule.profile.description == "Simple NIST Profile" + assert rule.profile.href == "profiles/simplified_nist_profile/profile.json" + + +def test_rules_transform_with_incomplete_rule() -> None: + """Test rules transform with incomplete rule.""" + # Generate test json string + test_string = '{"test_json": "test"}' + transformer = ToRulesYAMLTransformer() + + with pytest.raises( + RulesTransformerException, + match="Missing key in YAML file: 'x-trestle-rule-info'", + ): + transformer.transform(test_string) + + +def test_rules_transform_with_invalid_rule() -> None: + """Test rules transform with invalid rule.""" + # load rule from path and close the file + # get the file info as a string + rule_path = YAML_TEST_DATA_PATH / "test_invalid_rule.yaml" + rule_file = open(rule_path, "r") + rule_file_info = rule_file.read() + rule_file.close() + transformer = ToRulesYAMLTransformer() + + with pytest.raises( + RulesTransformerException, match="Invalid YAML file: 1 validation error .*" + ): + transformer.transform(rule_file_info) + + +def test_read_write_integration(test_rule: TrestleRule) -> None: + """Test read/write integration.""" + from_rules_transformer = FromRulesYAMLTransformer() + to_rules_transformer = ToRulesYAMLTransformer() + + yaml_data = from_rules_transformer.transform(test_rule) + + blob = yaml_data.getvalue() + + read_rule = to_rules_transformer.transform(blob) + + assert read_rule == test_rule diff --git a/trestlebot/const.py b/trestlebot/const.py index 5c6d2a37..f7ad568a 100644 --- a/trestlebot/const.py +++ b/trestlebot/const.py @@ -34,12 +34,11 @@ RULE_INFO_TAG = trestle_const.TRESTLE_TAG + "rule-info" NAME = "name" DESCRIPTION = "description" -PARAMETERS = "parameter" -PROFILES = "profile" +PARAMETER = "parameter" +PROFILE = "profile" HREF = "href" -INCLUDE_CONTROLS = "include-controls" -DEFAULT_VALUE = "default-value" ALTERNATIVE_VALUES = "alternative-values" +DEFAULT_VALUE = "default-value" COMPONENT_YAML = "component.yaml" COMPONENT_INFO_TAG = trestle_const.TRESTLE_TAG + "component-info" diff --git a/trestlebot/tasks/authored/compdef.py b/trestlebot/tasks/authored/compdef.py index 81935b9f..9c691855 100644 --- a/trestlebot/tasks/authored/compdef.py +++ b/trestlebot/tasks/authored/compdef.py @@ -182,7 +182,7 @@ def create_new_default( ruledir.parent.mkdir(parents=True, exist_ok=True) empty_yaml = YAMLBuilder() - empty_yaml.write_empty_trestle_rule_keys(ruledir) + empty_yaml.write_default_trestle_rule_keys(ruledir) def get_control_implementation( diff --git a/trestlebot/tasks/rule_transform_task.py b/trestlebot/tasks/rule_transform_task.py index b2ff5674..7944673e 100644 --- a/trestlebot/tasks/rule_transform_task.py +++ b/trestlebot/tasks/rule_transform_task.py @@ -28,11 +28,9 @@ import trestlebot.const as const from trestlebot.tasks.base_task import TaskBase, TaskException -from trestlebot.transformers.trestle_rule import ( - RulesTransformer, - RulesTransformerException, -) -from trestlebot.transformers.yaml_to_csv import CSVBuilder +from trestlebot.transformers.base_transformer import RulesTransformerException +from trestlebot.transformers.csv_transformer import CSVBuilder +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer logger = logging.getLogger(__name__) @@ -47,7 +45,7 @@ def __init__( self, working_dir: str, rules_view_dir: str, - rule_transformer: RulesTransformer, + rule_transformer: ToRulesYAMLTransformer, skip_model_list: List[str] = [], ) -> None: """ @@ -67,7 +65,7 @@ def __init__( """ self._rule_view_dir = rules_view_dir - self._rule_transformer: RulesTransformer = rule_transformer + self._rule_transformer: ToRulesYAMLTransformer = rule_transformer super().__init__(working_dir, skip_model_list) def execute(self) -> int: diff --git a/trestlebot/transformers/__init__.py b/trestlebot/transformers/__init__.py index 034efddd..d160aa73 100644 --- a/trestlebot/transformers/__init__.py +++ b/trestlebot/transformers/__init__.py @@ -17,4 +17,8 @@ Trestlebot transformers. Custom transformers using trestle libs for trestlebot. + +The RulesTransformer class is the base class for all transformers pertaining +to TrestleRule objects. This allows us to have a common interface for all formats +that express rules. """ diff --git a/trestlebot/transformers/base_transformer.py b/trestlebot/transformers/base_transformer.py new file mode 100644 index 00000000..bac21150 --- /dev/null +++ b/trestlebot/transformers/base_transformer.py @@ -0,0 +1,44 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Base transformer for rules.""" + +from abc import abstractmethod +from typing import Any + +from trestle.transforms.transformer_factory import TransformerBase + +from trestlebot.transformers.trestle_rule import TrestleRule + + +class ToRulesTransformer(TransformerBase): + """Abstract interface for transforming to rule data.""" + + @abstractmethod + def transform(self, data: Any) -> TrestleRule: + """Transform to rule data.""" + + +class FromRulesTransformer(TransformerBase): + """Abstract interface for transforming from rule data.""" + + @abstractmethod + def transform(self, rule: TrestleRule) -> Any: + """Transform from rule data.""" + + +class RulesTransformerException(Exception): + """An error during transformation of a rule""" diff --git a/trestlebot/transformers/csv_to_yaml.py b/trestlebot/transformers/csv_to_yaml.py index 025b4f40..f1d08172 100644 --- a/trestlebot/transformers/csv_to_yaml.py +++ b/trestlebot/transformers/csv_to_yaml.py @@ -13,30 +13,30 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +"""CSV to YAML converter for rule authoring.""" import csv -import json import pathlib -from dataclasses import asdict, fields -from typing import Dict, List, Optional +from typing import List -import ruamel.yaml as yaml -import trestle.tasks.csv_to_oscal_cd as csv_to_oscal_cd - -from trestlebot import const +from trestlebot.transformers.csv_transformer import ToRulesCSVTransformer from trestlebot.transformers.trestle_rule import ( ComponentInfo, Control, - Parameter, Profile, TrestleRule, ) +from trestlebot.transformers.yaml_transformer import FromRulesYAMLTransformer class YAMLBuilder: + """Build Rules View in YAML from a CSV file.""" + def __init__(self) -> None: """Initialize.""" self._rules: List[TrestleRule] = [] + self._yaml_transformer = FromRulesYAMLTransformer() + self._csv_transformer = ToRulesCSVTransformer() def read_from_csv(self, filepath: pathlib.Path) -> None: """Read from a CSV file and populate self._rules.""" @@ -44,81 +44,42 @@ def read_from_csv(self, filepath: pathlib.Path) -> None: with open(filepath, mode="r", newline="") as csv_file: reader = csv.DictReader(csv_file) for row in reader: - self._rules.append(self._csv_to_rule(row)) - except Exception as e: - raise CSVReadError(f"Failed to read from CSV file: {e}") - - def _csv_to_rule(self, row: Dict[str, str]) -> TrestleRule: - """Transform a CSV row to a TrestleRule object.""" - rule_info = self._extract_rule_info(row) - profile = self._extract_profile(row) - component_info = self._extract_component_info(row) - parameter = self._extract_parameter(row) - - return TrestleRule( - name=rule_info[const.NAME], - description=rule_info[const.DESCRIPTION], - component=component_info, - parameter=parameter, - profile=profile, - ) - - def _extract_rule_info(self, row: Dict[str, str]) -> Dict[str, str]: - """Extract rule information from a CSV row.""" - return { - "name": row.get(csv_to_oscal_cd.RULE_ID, ""), - "description": row.get(csv_to_oscal_cd.RULE_DESCRIPTION, ""), - } - - def _extract_profile(self, row: Dict[str, str]) -> Profile: - """Extract profile information from a CSV row.""" - controls_list = row.get(csv_to_oscal_cd.CONTROL_ID_LIST, "").split(", ") - return Profile( - description=row.get(csv_to_oscal_cd.PROFILE_DESCRIPTION, ""), - href=row.get(csv_to_oscal_cd.PROFILE_SOURCE, ""), - include_controls=[ - Control(id=control_id.strip()) for control_id in controls_list - ], - ) - - def _extract_parameter(self, row: Dict[str, str]) -> Optional[Parameter]: - """Extract parameter information from a CSV row.""" - parameter_name = row.get(csv_to_oscal_cd.PARAMETER_ID, None) - if parameter_name: - return Parameter( - name=parameter_name, - description=row.get(csv_to_oscal_cd.PARAMETER_DESCRIPTION, ""), - alternative_values=json.loads( - row.get(csv_to_oscal_cd.PARAMETER_VALUE_ALTERNATIVES, "{}") - ), - default_value=row.get(csv_to_oscal_cd.PARAMETER_VALUE_DEFAULT, ""), - ) - return None - - def _extract_component_info(self, row: Dict[str, str]) -> ComponentInfo: - """Extract component information from a CSV row.""" - return ComponentInfo( - name=row.get(csv_to_oscal_cd.COMPONENT_TITLE, ""), - type=row.get(csv_to_oscal_cd.COMPONENT_TYPE, ""), - description=row.get(csv_to_oscal_cd.COMPONENT_DESCRIPTION, ""), - ) + self._rules.append(self._csv_transformer.transform(row)) + except FileNotFoundError: + raise CSVReadError(f"File not found: {filepath}") + except csv.Error as e: + raise CSVReadError(f"CSV reading error: {e}") def write_to_yaml(self, filepath: pathlib.Path) -> None: """Write the rules to a YAML file.""" try: - with open(filepath, "w") as yaml_file: - yaml.dump( - [asdict(rule) for rule in self._rules], yaml_file - ) # Use Python's built-in asdict + with open(filepath, mode="w") as yaml_file: + yaml_file.write("---\n") + for rule in self._rules: + yaml_str = self._yaml_transformer.transform(rule).getvalue() + yaml_file.write(yaml_str) + yaml_file.write("\n") except Exception as e: raise YAMLWriteError(f"Failed to write rules to YAML file: {e}") - def write_empty_trestle_rule_keys(self, filepath: pathlib.Path) -> None: - """Write empty TrestleRule keys to a YAML file.""" + def write_default_trestle_rule_keys(self, filepath: pathlib.Path) -> None: + """Write default TrestleRule keys to a YAML file.""" try: - empty_dict = {f.name: "" for f in fields(TrestleRule)} - with open(filepath, "w") as yaml_file: - yaml.dump(empty_dict, yaml_file) + test_rule = TrestleRule( + name="example rule", + description="example description", + component=ComponentInfo( + name="example component", + type="service", + description="example description", + ), + profile=Profile( + description="example profile", + href="example href", + include_controls=[Control(id="example")], + ), + ) + self._yaml_transformer.write_to_file(test_rule, filepath) except Exception as e: raise YAMLWriteError( f"Failed to write empty TrestleRule keys to YAML file: {e}" diff --git a/trestlebot/transformers/yaml_to_csv.py b/trestlebot/transformers/csv_transformer.py similarity index 55% rename from trestlebot/transformers/yaml_to_csv.py rename to trestlebot/transformers/csv_transformer.py index c71fbc5a..f3bae64f 100644 --- a/trestlebot/transformers/yaml_to_csv.py +++ b/trestlebot/transformers/csv_transformer.py @@ -14,13 +14,15 @@ # License for the specific language governing permissions and limitations # under the License. -"""YAML to CSV transformer for rule authoring.""" +"""CSV Tranformer for rule authoring.""" + import csv +import json import logging import pathlib -from typing import Any, Dict, List +from typing import Dict, List, Optional -from ruamel.yaml import YAML +import trestle.tasks.csv_to_oscal_cd as csv_to_oscal_cd from trestle.common.const import TRESTLE_GENERIC_NS from trestle.tasks.csv_to_oscal_cd import ( COMPONENT_DESCRIPTION, @@ -40,13 +42,15 @@ ) from trestlebot import const +from trestlebot.transformers.base_transformer import ( + FromRulesTransformer, + ToRulesTransformer, +) from trestlebot.transformers.trestle_rule import ( ComponentInfo, Control, Parameter, Profile, - RulesTransformer, - RulesTransformerException, TrestleRule, ) @@ -54,94 +58,88 @@ logger = logging.getLogger(__name__) -class RulesYAMLTransformer(RulesTransformer): - """Interface for YAML transformer to Rules model.""" +class ToRulesCSVTransformer(ToRulesTransformer): + """ + Interface for CSV transformer to Rules model. + + Notes: This will transform individual rows of CSV to and from a + Trestle object with row compliance with the Trestle CSV requirements. + """ def __init__(self) -> None: """Initialize.""" super().__init__() - def transform(self, blob: str) -> TrestleRule: - """Rules YAML data into a row of CSV.""" - trestle_rule: TrestleRule = self._ingest_yaml(blob) - return trestle_rule - - @staticmethod - def _ingest_yaml(blob: str) -> TrestleRule: - """Ingest the YAML blob into a TrestleData object.""" - try: - yaml = YAML(typ="safe") - yaml_data: Dict[str, Any] = yaml.load(blob) - - rule_info_data = yaml_data[const.RULE_INFO_TAG] - - # Unpack profile data - profile_data = rule_info_data[const.PROFILES] - profile_instance: Profile = Profile( - description=profile_data[const.DESCRIPTION], - href=profile_data[const.HREF], - include_controls=[ - Control(**control) - for control in profile_data[const.INCLUDE_CONTROLS] - ], - ) - - component_info_data = yaml_data[const.COMPONENT_INFO_TAG] - component_info_instance: ComponentInfo = ComponentInfo( - **component_info_data - ) + def transform(self, row: Dict[str, str]) -> TrestleRule: + """Transform a CSV row to a TrestleRule object.""" + rule_info = self._extract_rule_info(row) + profile = self._extract_profile(row) + component_info = self._extract_component_info(row) + parameter = self._extract_parameter(row) + + return TrestleRule( + name=rule_info[const.NAME], + description=rule_info[const.DESCRIPTION], + component=component_info, + parameter=parameter, + profile=profile, + ) + + def _extract_rule_info(self, row: Dict[str, str]) -> Dict[str, str]: + """Extract rule information from a CSV row.""" + return { + "name": row.get(csv_to_oscal_cd.RULE_ID, ""), + "description": row.get(csv_to_oscal_cd.RULE_DESCRIPTION, ""), + } - rule_info_instance: TrestleRule = TrestleRule( - name=rule_info_data[const.NAME], - description=rule_info_data[const.DESCRIPTION], - component=component_info_instance, - parameter=None, - profile=profile_instance, + def _extract_profile(self, row: Dict[str, str]) -> Profile: + """Extract profile information from a CSV row.""" + controls_list = row.get(csv_to_oscal_cd.CONTROL_ID_LIST, "").split(", ") + return Profile( + description=row.get(csv_to_oscal_cd.PROFILE_DESCRIPTION, ""), + href=row.get(csv_to_oscal_cd.PROFILE_SOURCE, ""), + include_controls=[ + Control(id=control_id.strip()) for control_id in controls_list + ], + ) + + def _extract_parameter(self, row: Dict[str, str]) -> Optional[Parameter]: + """Extract parameter information from a CSV row.""" + parameter_name = row.get(csv_to_oscal_cd.PARAMETER_ID, None) + if parameter_name: + return Parameter( + name=parameter_name, + description=row.get(csv_to_oscal_cd.PARAMETER_DESCRIPTION, ""), + alternative_values=json.loads( + row.get(csv_to_oscal_cd.PARAMETER_VALUE_ALTERNATIVES, "{}") + ), + default_value=row.get(csv_to_oscal_cd.PARAMETER_VALUE_DEFAULT, ""), ) + return None - if const.PARAMETERS in rule_info_data: - parameter_data = rule_info_data[const.PARAMETERS] - parameter_instance: Parameter = Parameter( - name=parameter_data[const.NAME], - description=parameter_data[const.DESCRIPTION], - alternative_values=parameter_data[const.ALTERNATIVE_VALUES], - default_value=parameter_data[const.DEFAULT_VALUE], - ) - rule_info_instance.parameter = parameter_instance + def _extract_component_info(self, row: Dict[str, str]) -> ComponentInfo: + """Extract component information from a CSV row.""" + return ComponentInfo( + name=row.get(csv_to_oscal_cd.COMPONENT_TITLE, ""), + type=row.get(csv_to_oscal_cd.COMPONENT_TYPE, ""), + description=row.get(csv_to_oscal_cd.COMPONENT_DESCRIPTION, ""), + ) - except KeyError as e: - raise RulesTransformerException(f"Missing key in YAML file: {e}") - except Exception as e: - raise RuntimeError(e) - return rule_info_instance +class FromRulesCSVTransformer(FromRulesTransformer): + """ + Interface for CSV transformer from Rules model. + Notes: This will transform individual rows of CSV to and from a + Trestle object with row compliance with the Trestle CSV requirements. + """ -class CSVBuilder: def __init__(self) -> None: """Initialize.""" - self._csv_columns: CsvColumn = CsvColumn() - self._rows: List[Dict[str, str]] = [] - - @property - def row_count(self) -> int: - """Return the number of rows.""" - return len(self._rows) - - def add_row(self, rule: TrestleRule) -> None: - """Add a row to the CSV.""" - row = self._rule_to_csv(rule) - self.validate_row(row) - self._rows.append(row) - - def validate_row(self, row: Dict[str, str]) -> None: - """Validate a row.""" - for key in self._csv_columns.get_required_column_names(): - if key not in row: - raise RuntimeError(f"Row missing key: {key}") + super().__init__() - def _rule_to_csv(self, rule: TrestleRule) -> Dict[str, str]: - """Transform rules data to CSV.""" + def transform(self, rule: TrestleRule) -> Dict[str, str]: + """Transforms TrestleRule into a row of CSV.""" rule_dict: Dict[str, str] = { RULE_ID: rule.name, RULE_DESCRIPTION: rule.description, @@ -185,6 +183,33 @@ def _add_component_info(self, component_info: ComponentInfo) -> Dict[str, str]: } return comp_dict + +class CSVBuilder: + """Build a Trestle compliant CSV from a list of TrestleRules.""" + + def __init__(self) -> None: + """Initialize.""" + self._csv_columns: CsvColumn = CsvColumn() + self._transformer: FromRulesTransformer = FromRulesCSVTransformer() + self._rows: List[Dict[str, str]] = [] + + @property + def row_count(self) -> int: + """Return the number of rows.""" + return len(self._rows) + + def add_row(self, rule: TrestleRule) -> None: + """Add a row to the CSV.""" + row = self._transformer.transform(rule) + self.validate_row(row) + self._rows.append(row) + + def validate_row(self, row: Dict[str, str]) -> None: + """Validate a row.""" + for key in self._csv_columns.get_required_column_names(): + if key not in row: + raise RuntimeError(f"Row missing key: {key}") + def write_to_file(self, filepath: pathlib.Path) -> None: """Write the CSV to file.""" logger.debug(f"Writing CSV to {filepath}") diff --git a/trestlebot/transformers/trestle_rule.py b/trestlebot/transformers/trestle_rule.py index 19c0bf88..23be279a 100644 --- a/trestlebot/transformers/trestle_rule.py +++ b/trestlebot/transformers/trestle_rule.py @@ -14,43 +14,43 @@ # License for the specific language governing permissions and limitations # under the License. -"""Trestle Rule Dataclass and base transformer.""" +"""Trestle Rule class with pydantic.""" -from abc import abstractmethod -from dataclasses import dataclass from typing import Any, Dict, List, Optional -from trestle.transforms.transformer_factory import TransformerBase +from pydantic import BaseModel, Field -@dataclass -class Parameter: +class Parameter(BaseModel): """Parameter dataclass.""" name: str description: str - alternative_values: Dict[str, Any] - default_value: str + alternative_values: Dict[str, Any] = Field(..., alias="alternative-values") + default_value: str = Field(..., alias="default-value") + class Config: + allow_population_by_field_name = True -@dataclass -class Control: + +class Control(BaseModel): """Control dataclass.""" id: str -@dataclass -class Profile: +class Profile(BaseModel): """Profile dataclass.""" description: str href: str - include_controls: List[Control] + include_controls: List[Control] = Field(..., alias="include-controls") + + class Config: + allow_population_by_field_name = True -@dataclass -class ComponentInfo: +class ComponentInfo(BaseModel): """ComponentInfo dataclass.""" name: str @@ -58,8 +58,7 @@ class ComponentInfo: description: str -@dataclass -class TrestleRule: +class TrestleRule(BaseModel): """TrestleRule dataclass.""" name: str @@ -67,15 +66,3 @@ class TrestleRule: component: ComponentInfo parameter: Optional[Parameter] profile: Profile - - -class RulesTransformer(TransformerBase): - """Abstract interface for transformers for rules""" - - @abstractmethod - def transform(self, blob: str) -> TrestleRule: - """Transform rule data.""" - - -class RulesTransformerException(Exception): - """An error during transformation of a rule""" diff --git a/trestlebot/transformers/yaml_transformer.py b/trestlebot/transformers/yaml_transformer.py new file mode 100644 index 00000000..f03debf0 --- /dev/null +++ b/trestlebot/transformers/yaml_transformer.py @@ -0,0 +1,136 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""YAML transformer for rule authoring.""" +import logging +import pathlib +from io import StringIO +from typing import Any, Dict + +from pydantic import ValidationError +from ruamel.yaml import YAML + +from trestlebot import const +from trestlebot.transformers.base_transformer import ( + FromRulesTransformer, + RulesTransformerException, + ToRulesTransformer, +) +from trestlebot.transformers.trestle_rule import ( + ComponentInfo, + Parameter, + Profile, + TrestleRule, +) + + +logger = logging.getLogger(__name__) + + +class ToRulesYAMLTransformer(ToRulesTransformer): + """Interface for YAML transformer to Rules model.""" + + def __init__(self) -> None: + """Initialize.""" + super().__init__() + + def transform(self, blob: str) -> TrestleRule: + """Transform YAML data into a TrestleRule object.""" + try: + yaml = YAML(typ="safe") + yaml_data: Dict[str, Any] = yaml.load(blob) + + rule_info_data = yaml_data[const.RULE_INFO_TAG] + + profile_data = rule_info_data[const.PROFILE] + profile_info_instance: Profile = Profile.parse_obj(profile_data) + + component_info_data = yaml_data[const.COMPONENT_INFO_TAG] + component_info_instance: ComponentInfo = ComponentInfo.parse_obj( + component_info_data + ) + + rule_info_instance: TrestleRule = TrestleRule( + name=rule_info_data[const.NAME], + description=rule_info_data[const.DESCRIPTION], + component=component_info_instance, + parameter=None, + profile=profile_info_instance, + ) + + if const.PARAMETER in rule_info_data: + parameter_data = rule_info_data[const.PARAMETER] + parameter_instance: Parameter = Parameter.parse_obj(parameter_data) + rule_info_instance.parameter = parameter_instance + + except KeyError as e: + raise RulesTransformerException(f"Missing key in YAML file: {e}") + except ValidationError as e: + raise RulesTransformerException(f"Invalid YAML file: {e}") + + return rule_info_instance + + +class FromRulesYAMLTransformer(FromRulesTransformer): + """ + Interface for YAML transformer from Rules model. + """ + + def transform(self, rule: TrestleRule) -> StringIO: + """ + Transform TrestleRule object into YAML data. + + Notes: + Currently, this method is for simply transforming + TrestleRule objects in memory into YAML data. The data structure + is small so this should have minimal performance impact. This is done to + comply with the contract of the RulesTransformer interface. + """ + + rule_info: Dict[str, Any] = self._to_rule_info(rule) + yaml_obj = YAML() + yaml_obj.default_flow_style = False + yaml_stream = StringIO() + yaml_obj.dump(rule_info, yaml_stream) + + return yaml_stream + + def write_to_file(self, rule: TrestleRule, file_path: pathlib.Path) -> None: + """Write TrestleRule object to YAML file.""" + rule_info: Dict[str, Any] = self._to_rule_info(rule) + yaml_obj = YAML() + yaml_obj.default_flow_style = False + yaml_obj.dump(rule_info, file_path) + + @staticmethod + def _to_rule_info(rule: TrestleRule) -> Dict[str, Any]: + """Convert YAML data to rule info.""" + rule_info: Dict[str, Any] = { + const.RULE_INFO_TAG: { + const.NAME: rule.name, + const.DESCRIPTION: rule.description, + const.PROFILE: rule.profile.dict(by_alias=True, exclude_unset=True), + }, + const.COMPONENT_INFO_TAG: rule.component.dict( + by_alias=True, exclude_unset=True + ), + } + + if rule.parameter is not None: + rule_info[const.RULE_INFO_TAG][const.PARAMETER] = rule.parameter.dict( + by_alias=True, exclude_unset=True + ) + return rule_info