From 4f8d388c3aefb1ee5b57dca681ceaa3e805ba3ce Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Tue, 3 Oct 2023 19:10:17 -0400 Subject: [PATCH 1/6] refactor: reworks csv_to_yaml for pydantic Update write empty keys to write default keys Create transformer for CSV reading and writing Moves CSV specific logic t csv_transformer Moves YAML specific logic to yaml_transformer and adds writing logic Signed-off-by: Jennifer Power --- poetry.lock | 161 ++++++++--------- pyproject.toml | 1 + tests/data/yaml/test_complete_rule.yaml | 6 +- .../yaml/test_complete_rule_no_params.yaml | 2 +- tests/data/yaml/test_incomplete_rule.yaml | 6 +- tests/data/yaml/test_invalid_rule.yaml | 17 ++ .../tasks/test_rule_transform_task.py | 2 +- .../transformers/test_csv_to_yaml.py | 23 ++- ...yaml_to_csv.py => test_csv_transformer.py} | 56 +----- .../transformers/test_yaml_transformer.py | 84 +++++++++ trestlebot/const.py | 4 +- trestlebot/tasks/authored/compdef.py | 2 +- trestlebot/tasks/rule_transform_task.py | 6 +- trestlebot/transformers/base_transformer.py | 38 ++++ trestlebot/transformers/csv_to_yaml.py | 97 +++------- .../{yaml_to_csv.py => csv_transformer.py} | 170 +++++++++--------- trestlebot/transformers/trestle_rule.py | 33 +--- trestlebot/transformers/yaml_transformer.py | 119 ++++++++++++ 18 files changed, 487 insertions(+), 340 deletions(-) create mode 100644 tests/data/yaml/test_invalid_rule.yaml rename tests/trestlebot/transformers/{test_yaml_to_csv.py => test_csv_transformer.py} (61%) create mode 100644 tests/trestlebot/transformers/test_yaml_transformer.py create mode 100644 trestlebot/transformers/base_transformer.py rename trestlebot/transformers/{yaml_to_csv.py => csv_transformer.py} (60%) create mode 100644 trestlebot/transformers/yaml_transformer.py diff --git a/poetry.lock b/poetry.lock index 32602522..cd7d2046 100644 --- a/poetry.lock +++ b/poetry.lock @@ -430,7 +430,7 @@ files = [ [[package]] name = "compliance-trestle" -version = "0.1.dev1134+g27533b4" +version = "2.3.1.dev29+g725f6980" description = "Tools to manage & autogenerate python objects representing the OSCAL layers/models" optional = false python-versions = "*" @@ -1518,52 +1518,52 @@ files = [ [[package]] name = "pydantic" -version = "1.10.13" +version = "1.10.2" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, - {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, - {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, - {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, - {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, - {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, - {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, - {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, - {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, - {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, + {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, + {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, + {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, + {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, + {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, + {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, + {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, + {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] [package.dependencies] email-validator = {version = ">=1.0.3", optional = true, markers = "extra == \"email\""} -typing-extensions = ">=4.2.0" +typing-extensions = ">=4.1.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -1830,7 +1830,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1838,15 +1837,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1863,7 +1855,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1871,7 +1862,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1965,48 +1955,41 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] [[package]] name = "ruamel-yaml-clib" -version = "0.2.7" +version = "0.2.8" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:370445fd795706fd291ab00c9df38a0caed0f17a6fb46b0f607668ecb16ce763"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win32.whl", hash = "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:40d030e2329ce5286d6b231b8726959ebbe0404c92f0a578c0e2482182e38282"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c3ca1fbba4ae962521e5eb66d72998b51f0f4d0f608d3c0347a48e1af262efa7"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3243f48ecd450eddadc2d11b5feb08aca941b5cd98c9b1db14b2fd128be8c697"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win32.whl", hash = "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf9a6bc4a0221538b1a7de3ed7bca4c93c02346853f44e1cd764be0023cd3640"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win32.whl", hash = "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5"}, - {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, + {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, ] [[package]] @@ -2177,4 +2160,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "b6d05c2abf33d5cb66d9adaaa077061fea617623672e5ba69148494073b2cfcb" +content-hash = "d2725065f43dfc637e8ce40a2e6c9b9587b1ec3b01a85353acfb9c60a4e4ff1d" diff --git a/pyproject.toml b/pyproject.toml index 76eba6ba..a28c9c94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ compliance-trestle = {git = "https://github.com/IBM/compliance-trestle.git", rev github3-py = "^4.0.1" python-gitlab = "^3.15.0" ruamel-yaml = "^0.17.32" +pydantic = "1.10.2" [tool.poetry.group.dev.dependencies] flake8 = "^6.0.0" diff --git a/tests/data/yaml/test_complete_rule.yaml b/tests/data/yaml/test_complete_rule.yaml index 2a37605a..9f41c19f 100644 --- a/tests/data/yaml/test_complete_rule.yaml +++ b/tests/data/yaml/test_complete_rule.yaml @@ -4,12 +4,12 @@ x-trestle-rule-info: parameter: name: prm_1 description: prm_1 description - alternative-values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} - default-value: '5%' + alternative_values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} + default_value: '5%' profile: description: Simple NIST Profile href: profiles/simplified_nist_profile/profile.json - include-controls: + include_controls: - id: ac-1 x-trestle-component-info: name: Component 1 diff --git a/tests/data/yaml/test_complete_rule_no_params.yaml b/tests/data/yaml/test_complete_rule_no_params.yaml index f95d72f7..0e005f3c 100644 --- a/tests/data/yaml/test_complete_rule_no_params.yaml +++ b/tests/data/yaml/test_complete_rule_no_params.yaml @@ -4,7 +4,7 @@ x-trestle-rule-info: profile: description: Simple NIST Profile href: profiles/simplified_nist_profile/profile.json - include-controls: + include_controls: - id: ac-1 x-trestle-component-info: name: Component 1 diff --git a/tests/data/yaml/test_incomplete_rule.yaml b/tests/data/yaml/test_incomplete_rule.yaml index e8e2ec33..ad368ac9 100644 --- a/tests/data/yaml/test_incomplete_rule.yaml +++ b/tests/data/yaml/test_incomplete_rule.yaml @@ -3,12 +3,12 @@ x-trestle-rule-info: parameter: name: prm_1 description: prm_1 description - alternative-values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} - default-value: '5%' + alternative_values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} + default_value: '5%' profile: description: Simple NIST Profile href: profiles/simplified_nist_profile/profile.json - include-controls: + include_controls: - id: ac-2 x-trestle-component-info: name: Component 1 diff --git a/tests/data/yaml/test_invalid_rule.yaml b/tests/data/yaml/test_invalid_rule.yaml new file mode 100644 index 00000000..cf6cb98e --- /dev/null +++ b/tests/data/yaml/test_invalid_rule.yaml @@ -0,0 +1,17 @@ +x-trestle-rule-info: + name: example_rule_1 + description: My rule description for example rule 1 + parameter: + name: prm_1 + description: prm_1 description + alternative_values: "invalid" + default_value: true + profile: + description: Simple NIST Profile + href: profiles/simplified_nist_profile/profile.json + include_controls: + - id: ac-2 +x-trestle-component-info: + name: Component 1 + description: Component 1 description + type: service diff --git a/tests/trestlebot/tasks/test_rule_transform_task.py b/tests/trestlebot/tasks/test_rule_transform_task.py index 59c75464..37869144 100644 --- a/tests/trestlebot/tasks/test_rule_transform_task.py +++ b/tests/trestlebot/tasks/test_rule_transform_task.py @@ -28,7 +28,7 @@ from tests.testutils import setup_rules_view from trestlebot.tasks.base_task import TaskException from trestlebot.tasks.rule_transform_task import RuleTransformTask -from trestlebot.transformers.yaml_to_csv import RulesYAMLTransformer +from trestlebot.transformers.yaml_transformer import RulesYAMLTransformer test_comp = "test_comp" diff --git a/tests/trestlebot/transformers/test_csv_to_yaml.py b/tests/trestlebot/transformers/test_csv_to_yaml.py index 290d295a..d85f177e 100644 --- a/tests/trestlebot/transformers/test_csv_to_yaml.py +++ b/tests/trestlebot/transformers/test_csv_to_yaml.py @@ -16,13 +16,12 @@ import csv import pathlib -from dataclasses import fields import pytest import ruamel.yaml as yaml from trestlebot.transformers.csv_to_yaml import YAMLBuilder -from trestlebot.transformers.trestle_rule import TrestleRule +from trestlebot.transformers.yaml_transformer import RulesYAMLTransformer @pytest.fixture(scope="function") @@ -76,13 +75,19 @@ def test_write_to_yaml(setup_yaml_builder: YAMLBuilder, tmp_trestle_dir: str) -> assert len(data) == 1 -def test_write_empty_trestle_rule_keys( +def test_default_test_trestle_rule_keys( setup_yaml_builder: YAMLBuilder, tmp_trestle_dir: str ) -> None: yaml_file = pathlib.Path(tmp_trestle_dir) / "test.yaml" - setup_yaml_builder.write_empty_trestle_rule_keys(yaml_file) - with open(yaml_file, "r") as f: - data = yaml.safe_load(f) - assert all(value == "" for value in data.values()) - expected_keys = {field.name for field in fields(TrestleRule)} - assert expected_keys == set(data.keys()) + setup_yaml_builder.write_default_trestle_rule_keys(yaml_file) + transformer = RulesYAMLTransformer() + rule = transformer.transform_to_rule(yaml_file.read_text()) + + assert rule.name == "example rule" + assert rule.description == "example description" + assert rule.component.name == "example component" + assert rule.component.description == "example description" + assert rule.component.type == "service" + assert rule.profile.description == "example profile" + assert rule.profile.href == "example href" + assert len(rule.profile.include_controls) == 1 diff --git a/tests/trestlebot/transformers/test_yaml_to_csv.py b/tests/trestlebot/transformers/test_csv_transformer.py similarity index 61% rename from tests/trestlebot/transformers/test_yaml_to_csv.py rename to tests/trestlebot/transformers/test_csv_transformer.py index ef7e0f6a..b3d595d9 100644 --- a/tests/trestlebot/transformers/test_yaml_to_csv.py +++ b/tests/trestlebot/transformers/test_csv_transformer.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Test for YAML to CSV Transformer.""" +"""Test for CSV Transformer.""" import csv import pathlib @@ -22,16 +22,14 @@ import pytest -from tests.testutils import YAML_TEST_DATA_PATH +from trestlebot.transformers.csv_transformer import CSVBuilder from trestlebot.transformers.trestle_rule import ( ComponentInfo, Control, Parameter, Profile, - RulesTransformerException, TrestleRule, ) -from trestlebot.transformers.yaml_to_csv import CSVBuilder, RulesYAMLTransformer test_comp = "my comp" @@ -44,7 +42,10 @@ def test_csv_builder(tmp_trestle_dir: str) -> None: description="test", component=ComponentInfo(name=test_comp, type="test", description="test"), parameter=Parameter( - name="test", description="test", alternative_values={}, default_value="test" + name="test", + description="test", + alternative_values={}, + default_value="test", ), profile=Profile( description="test", href="test", include_controls=[Control(id="ac-1")] @@ -89,48 +90,3 @@ def test_validate_row() -> None: csv_builder = CSVBuilder() with pytest.raises(RuntimeError, match="Row missing key: *"): csv_builder.validate_row(row) - - -def test_rule_transformer() -> None: - """Test rule transformer.""" - # load rule from path and close the file - # get the file info as a string - rule_path = YAML_TEST_DATA_PATH / "test_complete_rule.yaml" - rule_file = open(rule_path, "r") - rule_file_info = rule_file.read() - rule_file.close() - - transformer = RulesYAMLTransformer() - rule = transformer.transform(rule_file_info) - - assert rule.name == "example_rule_1" - assert rule.description == "My rule description for example rule 1" - assert rule.component.name == "Component 1" - assert rule.component.type == "service" - assert rule.component.description == "Component 1 description" - assert rule.parameter is not None - assert rule.parameter.name == "prm_1" - assert rule.parameter.description == "prm_1 description" - assert rule.parameter.alternative_values == { - "default": "5%", - "5pc": "5%", - "10pc": "10%", - "15pc": "15%", - "20pc": "20%", - } - assert rule.parameter.default_value == "5%" - assert rule.profile.description == "Simple NIST Profile" - assert rule.profile.href == "profiles/simplified_nist_profile/profile.json" - - -def test_rules_transform_with_invalid_rule() -> None: - """Test rules transform with invalid rule.""" - # Generate test json string - test_string = '{"test_json": "test"}' - transformer = RulesYAMLTransformer() - - with pytest.raises( - RulesTransformerException, - match="Missing key in YAML file: 'x-trestle-rule-info'", - ): - transformer.transform(test_string) diff --git a/tests/trestlebot/transformers/test_yaml_transformer.py b/tests/trestlebot/transformers/test_yaml_transformer.py new file mode 100644 index 00000000..c393eae5 --- /dev/null +++ b/tests/trestlebot/transformers/test_yaml_transformer.py @@ -0,0 +1,84 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Test for YAML Transformer.""" + +import pytest + +from tests.testutils import YAML_TEST_DATA_PATH +from trestlebot.transformers.base_transformer import RulesTransformerException +from trestlebot.transformers.yaml_transformer import RulesYAMLTransformer + + +def test_rule_transformer() -> None: + """Test rule transformer.""" + # load rule from path and close the file + # get the file info as a string + rule_path = YAML_TEST_DATA_PATH / "test_complete_rule.yaml" + rule_file = open(rule_path, "r") + rule_file_info = rule_file.read() + rule_file.close() + + transformer = RulesYAMLTransformer() + rule = transformer.transform_to_rule(rule_file_info) + + assert rule.name == "example_rule_1" + assert rule.description == "My rule description for example rule 1" + assert rule.component.name == "Component 1" + assert rule.component.type == "service" + assert rule.component.description == "Component 1 description" + assert rule.parameter is not None + assert rule.parameter.name == "prm_1" + assert rule.parameter.description == "prm_1 description" + assert rule.parameter.alternative_values == { + "default": "5%", + "5pc": "5%", + "10pc": "10%", + "15pc": "15%", + "20pc": "20%", + } + assert rule.parameter.default_value == "5%" + assert rule.profile.description == "Simple NIST Profile" + assert rule.profile.href == "profiles/simplified_nist_profile/profile.json" + + +def test_rules_transform_with_incomplete_rule() -> None: + """Test rules transform with incomplete rule.""" + # Generate test json string + test_string = '{"test_json": "test"}' + transformer = RulesYAMLTransformer() + + with pytest.raises( + RulesTransformerException, + match="Missing key in YAML file: 'x-trestle-rule-info'", + ): + transformer.transform_to_rule(test_string) + + +def test_rules_transform_with_invalid_rule() -> None: + """Test rules transform with invalid rule.""" + # load rule from path and close the file + # get the file info as a string + rule_path = YAML_TEST_DATA_PATH / "test_invalid_rule.yaml" + rule_file = open(rule_path, "r") + rule_file_info = rule_file.read() + rule_file.close() + transformer = RulesYAMLTransformer() + + with pytest.raises( + RulesTransformerException, match="Invalid YAML file: 1 validation error .*" + ): + transformer.transform_to_rule(rule_file_info) diff --git a/trestlebot/const.py b/trestlebot/const.py index 5c6d2a37..5d905e80 100644 --- a/trestlebot/const.py +++ b/trestlebot/const.py @@ -34,8 +34,8 @@ RULE_INFO_TAG = trestle_const.TRESTLE_TAG + "rule-info" NAME = "name" DESCRIPTION = "description" -PARAMETERS = "parameter" -PROFILES = "profile" +PARAMETER = "parameter" +PROFILE = "profile" HREF = "href" INCLUDE_CONTROLS = "include-controls" DEFAULT_VALUE = "default-value" diff --git a/trestlebot/tasks/authored/compdef.py b/trestlebot/tasks/authored/compdef.py index 81935b9f..9c691855 100644 --- a/trestlebot/tasks/authored/compdef.py +++ b/trestlebot/tasks/authored/compdef.py @@ -182,7 +182,7 @@ def create_new_default( ruledir.parent.mkdir(parents=True, exist_ok=True) empty_yaml = YAMLBuilder() - empty_yaml.write_empty_trestle_rule_keys(ruledir) + empty_yaml.write_default_trestle_rule_keys(ruledir) def get_control_implementation( diff --git a/trestlebot/tasks/rule_transform_task.py b/trestlebot/tasks/rule_transform_task.py index b2ff5674..423f915a 100644 --- a/trestlebot/tasks/rule_transform_task.py +++ b/trestlebot/tasks/rule_transform_task.py @@ -28,11 +28,11 @@ import trestlebot.const as const from trestlebot.tasks.base_task import TaskBase, TaskException -from trestlebot.transformers.trestle_rule import ( +from trestlebot.transformers.base_transformer import ( RulesTransformer, RulesTransformerException, ) -from trestlebot.transformers.yaml_to_csv import CSVBuilder +from trestlebot.transformers.csv_transformer import CSVBuilder logger = logging.getLogger(__name__) @@ -101,7 +101,7 @@ def _transform_components(self, component_definition_path: pathlib.Path) -> None rule_stream = rule_path.read_text() try: - rule = self._rule_transformer.transform(rule_stream) + rule = self._rule_transformer.transform_to_rule(rule_stream) csv_builder.add_row(rule) except RulesTransformerException as e: raise TaskException( diff --git a/trestlebot/transformers/base_transformer.py b/trestlebot/transformers/base_transformer.py new file mode 100644 index 00000000..3d7538b0 --- /dev/null +++ b/trestlebot/transformers/base_transformer.py @@ -0,0 +1,38 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Base transformer for rules.""" + +from abc import ABC, abstractmethod +from typing import Any + +from trestlebot.transformers.trestle_rule import TrestleRule + + +class RulesTransformer(ABC): + """Abstract interface for transformers for rules""" + + @abstractmethod + def transform_to_rule(self, data: Any) -> TrestleRule: + """Transform to rule data.""" + + @abstractmethod + def transform_from_rule(self, rule: TrestleRule) -> Any: + """Transform from rule data.""" + + +class RulesTransformerException(Exception): + """An error during transformation of a rule""" diff --git a/trestlebot/transformers/csv_to_yaml.py b/trestlebot/transformers/csv_to_yaml.py index 025b4f40..d1ab2ce0 100644 --- a/trestlebot/transformers/csv_to_yaml.py +++ b/trestlebot/transformers/csv_to_yaml.py @@ -15,28 +15,29 @@ # under the License. import csv -import json import pathlib -from dataclasses import asdict, fields -from typing import Dict, List, Optional +from typing import List import ruamel.yaml as yaml -import trestle.tasks.csv_to_oscal_cd as csv_to_oscal_cd -from trestlebot import const +from trestlebot.transformers.csv_transformer import RulesCSVTransformer from trestlebot.transformers.trestle_rule import ( ComponentInfo, Control, - Parameter, Profile, TrestleRule, ) +from trestlebot.transformers.yaml_transformer import RulesYAMLTransformer class YAMLBuilder: + """Build Rules View in YAML from a CSV file.""" + def __init__(self) -> None: """Initialize.""" self._rules: List[TrestleRule] = [] + self._yaml_transformer = RulesYAMLTransformer() + self._csv_transformer = RulesCSVTransformer() def read_from_csv(self, filepath: pathlib.Path) -> None: """Read from a CSV file and populate self._rules.""" @@ -44,81 +45,41 @@ def read_from_csv(self, filepath: pathlib.Path) -> None: with open(filepath, mode="r", newline="") as csv_file: reader = csv.DictReader(csv_file) for row in reader: - self._rules.append(self._csv_to_rule(row)) + self._rules.append(self._csv_transformer.transform_to_rule(row)) except Exception as e: raise CSVReadError(f"Failed to read from CSV file: {e}") - def _csv_to_rule(self, row: Dict[str, str]) -> TrestleRule: - """Transform a CSV row to a TrestleRule object.""" - rule_info = self._extract_rule_info(row) - profile = self._extract_profile(row) - component_info = self._extract_component_info(row) - parameter = self._extract_parameter(row) - - return TrestleRule( - name=rule_info[const.NAME], - description=rule_info[const.DESCRIPTION], - component=component_info, - parameter=parameter, - profile=profile, - ) - - def _extract_rule_info(self, row: Dict[str, str]) -> Dict[str, str]: - """Extract rule information from a CSV row.""" - return { - "name": row.get(csv_to_oscal_cd.RULE_ID, ""), - "description": row.get(csv_to_oscal_cd.RULE_DESCRIPTION, ""), - } - - def _extract_profile(self, row: Dict[str, str]) -> Profile: - """Extract profile information from a CSV row.""" - controls_list = row.get(csv_to_oscal_cd.CONTROL_ID_LIST, "").split(", ") - return Profile( - description=row.get(csv_to_oscal_cd.PROFILE_DESCRIPTION, ""), - href=row.get(csv_to_oscal_cd.PROFILE_SOURCE, ""), - include_controls=[ - Control(id=control_id.strip()) for control_id in controls_list - ], - ) - - def _extract_parameter(self, row: Dict[str, str]) -> Optional[Parameter]: - """Extract parameter information from a CSV row.""" - parameter_name = row.get(csv_to_oscal_cd.PARAMETER_ID, None) - if parameter_name: - return Parameter( - name=parameter_name, - description=row.get(csv_to_oscal_cd.PARAMETER_DESCRIPTION, ""), - alternative_values=json.loads( - row.get(csv_to_oscal_cd.PARAMETER_VALUE_ALTERNATIVES, "{}") - ), - default_value=row.get(csv_to_oscal_cd.PARAMETER_VALUE_DEFAULT, ""), - ) - return None - - def _extract_component_info(self, row: Dict[str, str]) -> ComponentInfo: - """Extract component information from a CSV row.""" - return ComponentInfo( - name=row.get(csv_to_oscal_cd.COMPONENT_TITLE, ""), - type=row.get(csv_to_oscal_cd.COMPONENT_TYPE, ""), - description=row.get(csv_to_oscal_cd.COMPONENT_DESCRIPTION, ""), - ) - def write_to_yaml(self, filepath: pathlib.Path) -> None: """Write the rules to a YAML file.""" try: with open(filepath, "w") as yaml_file: yaml.dump( - [asdict(rule) for rule in self._rules], yaml_file - ) # Use Python's built-in asdict + [self._yaml_transformer._write_yaml(rule) for rule in self._rules], + yaml_file, + ) except Exception as e: raise YAMLWriteError(f"Failed to write rules to YAML file: {e}") - def write_empty_trestle_rule_keys(self, filepath: pathlib.Path) -> None: - """Write empty TrestleRule keys to a YAML file.""" + def write_default_trestle_rule_keys(self, filepath: pathlib.Path) -> None: + """Write default TrestleRule keys to a YAML file.""" try: - empty_dict = {f.name: "" for f in fields(TrestleRule)} + test_rule = TrestleRule( + name="example rule", + description="example description", + component=ComponentInfo( + name="example component", + type="service", + description="example description", + ), + profile=Profile( + description="example profile", + href="example href", + include_controls=[Control(id="example")], + ), + ) + yaml_blob = self._yaml_transformer._write_yaml(test_rule) with open(filepath, "w") as yaml_file: - yaml.dump(empty_dict, yaml_file) + yaml_file.write(yaml_blob) except Exception as e: raise YAMLWriteError( f"Failed to write empty TrestleRule keys to YAML file: {e}" diff --git a/trestlebot/transformers/yaml_to_csv.py b/trestlebot/transformers/csv_transformer.py similarity index 60% rename from trestlebot/transformers/yaml_to_csv.py rename to trestlebot/transformers/csv_transformer.py index c71fbc5a..5c39c550 100644 --- a/trestlebot/transformers/yaml_to_csv.py +++ b/trestlebot/transformers/csv_transformer.py @@ -14,13 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. -"""YAML to CSV transformer for rule authoring.""" import csv +import json import logging import pathlib -from typing import Any, Dict, List +from typing import Dict, List, Optional -from ruamel.yaml import YAML +import trestle.tasks.csv_to_oscal_cd as csv_to_oscal_cd from trestle.common.const import TRESTLE_GENERIC_NS from trestle.tasks.csv_to_oscal_cd import ( COMPONENT_DESCRIPTION, @@ -40,13 +40,12 @@ ) from trestlebot import const +from trestlebot.transformers.base_transformer import RulesTransformer from trestlebot.transformers.trestle_rule import ( ComponentInfo, Control, Parameter, Profile, - RulesTransformer, - RulesTransformerException, TrestleRule, ) @@ -54,94 +53,70 @@ logger = logging.getLogger(__name__) -class RulesYAMLTransformer(RulesTransformer): - """Interface for YAML transformer to Rules model.""" +class RulesCSVTransformer(RulesTransformer): + """Interface for CSV transformer to Rules model.""" def __init__(self) -> None: """Initialize.""" super().__init__() - def transform(self, blob: str) -> TrestleRule: - """Rules YAML data into a row of CSV.""" - trestle_rule: TrestleRule = self._ingest_yaml(blob) - return trestle_rule - - @staticmethod - def _ingest_yaml(blob: str) -> TrestleRule: - """Ingest the YAML blob into a TrestleData object.""" - try: - yaml = YAML(typ="safe") - yaml_data: Dict[str, Any] = yaml.load(blob) - - rule_info_data = yaml_data[const.RULE_INFO_TAG] - - # Unpack profile data - profile_data = rule_info_data[const.PROFILES] - profile_instance: Profile = Profile( - description=profile_data[const.DESCRIPTION], - href=profile_data[const.HREF], - include_controls=[ - Control(**control) - for control in profile_data[const.INCLUDE_CONTROLS] - ], - ) - - component_info_data = yaml_data[const.COMPONENT_INFO_TAG] - component_info_instance: ComponentInfo = ComponentInfo( - **component_info_data - ) + def transform_to_rule(self, row: Dict[str, str]) -> TrestleRule: + """Transform a CSV row to a TrestleRule object.""" + rule_info = self._extract_rule_info(row) + profile = self._extract_profile(row) + component_info = self._extract_component_info(row) + parameter = self._extract_parameter(row) + + return TrestleRule( + name=rule_info[const.NAME], + description=rule_info[const.DESCRIPTION], + component=component_info, + parameter=parameter, + profile=profile, + ) + + def _extract_rule_info(self, row: Dict[str, str]) -> Dict[str, str]: + """Extract rule information from a CSV row.""" + return { + "name": row.get(csv_to_oscal_cd.RULE_ID, ""), + "description": row.get(csv_to_oscal_cd.RULE_DESCRIPTION, ""), + } - rule_info_instance: TrestleRule = TrestleRule( - name=rule_info_data[const.NAME], - description=rule_info_data[const.DESCRIPTION], - component=component_info_instance, - parameter=None, - profile=profile_instance, + def _extract_profile(self, row: Dict[str, str]) -> Profile: + """Extract profile information from a CSV row.""" + controls_list = row.get(csv_to_oscal_cd.CONTROL_ID_LIST, "").split(", ") + return Profile( + description=row.get(csv_to_oscal_cd.PROFILE_DESCRIPTION, ""), + href=row.get(csv_to_oscal_cd.PROFILE_SOURCE, ""), + include_controls=[ + Control(id=control_id.strip()) for control_id in controls_list + ], + ) + + def _extract_parameter(self, row: Dict[str, str]) -> Optional[Parameter]: + """Extract parameter information from a CSV row.""" + parameter_name = row.get(csv_to_oscal_cd.PARAMETER_ID, None) + if parameter_name: + return Parameter( + name=parameter_name, + description=row.get(csv_to_oscal_cd.PARAMETER_DESCRIPTION, ""), + alternative_values=json.loads( + row.get(csv_to_oscal_cd.PARAMETER_VALUE_ALTERNATIVES, "{}") + ), + default_value=row.get(csv_to_oscal_cd.PARAMETER_VALUE_DEFAULT, ""), ) + return None - if const.PARAMETERS in rule_info_data: - parameter_data = rule_info_data[const.PARAMETERS] - parameter_instance: Parameter = Parameter( - name=parameter_data[const.NAME], - description=parameter_data[const.DESCRIPTION], - alternative_values=parameter_data[const.ALTERNATIVE_VALUES], - default_value=parameter_data[const.DEFAULT_VALUE], - ) - rule_info_instance.parameter = parameter_instance - - except KeyError as e: - raise RulesTransformerException(f"Missing key in YAML file: {e}") - except Exception as e: - raise RuntimeError(e) - - return rule_info_instance - - -class CSVBuilder: - def __init__(self) -> None: - """Initialize.""" - self._csv_columns: CsvColumn = CsvColumn() - self._rows: List[Dict[str, str]] = [] - - @property - def row_count(self) -> int: - """Return the number of rows.""" - return len(self._rows) - - def add_row(self, rule: TrestleRule) -> None: - """Add a row to the CSV.""" - row = self._rule_to_csv(rule) - self.validate_row(row) - self._rows.append(row) + def _extract_component_info(self, row: Dict[str, str]) -> ComponentInfo: + """Extract component information from a CSV row.""" + return ComponentInfo( + name=row.get(csv_to_oscal_cd.COMPONENT_TITLE, ""), + type=row.get(csv_to_oscal_cd.COMPONENT_TYPE, ""), + description=row.get(csv_to_oscal_cd.COMPONENT_DESCRIPTION, ""), + ) - def validate_row(self, row: Dict[str, str]) -> None: - """Validate a row.""" - for key in self._csv_columns.get_required_column_names(): - if key not in row: - raise RuntimeError(f"Row missing key: {key}") - - def _rule_to_csv(self, rule: TrestleRule) -> Dict[str, str]: - """Transform rules data to CSV.""" + def transform_from_rule(self, rule: TrestleRule) -> Dict[str, str]: + """Rules YAML data into a row of CSV.""" rule_dict: Dict[str, str] = { RULE_ID: rule.name, RULE_DESCRIPTION: rule.description, @@ -185,6 +160,33 @@ def _add_component_info(self, component_info: ComponentInfo) -> Dict[str, str]: } return comp_dict + +class CSVBuilder: + """Build a Trestle compliant CSV from a list of TrestleRules.""" + + def __init__(self) -> None: + """Initialize.""" + self._csv_columns: CsvColumn = CsvColumn() + self._transformer: RulesCSVTransformer = RulesCSVTransformer() + self._rows: List[Dict[str, str]] = [] + + @property + def row_count(self) -> int: + """Return the number of rows.""" + return len(self._rows) + + def add_row(self, rule: TrestleRule) -> None: + """Add a row to the CSV.""" + row = self._transformer.transform_from_rule(rule) + self.validate_row(row) + self._rows.append(row) + + def validate_row(self, row: Dict[str, str]) -> None: + """Validate a row.""" + for key in self._csv_columns.get_required_column_names(): + if key not in row: + raise RuntimeError(f"Row missing key: {key}") + def write_to_file(self, filepath: pathlib.Path) -> None: """Write the CSV to file.""" logger.debug(f"Writing CSV to {filepath}") diff --git a/trestlebot/transformers/trestle_rule.py b/trestlebot/transformers/trestle_rule.py index 19c0bf88..87630ae9 100644 --- a/trestlebot/transformers/trestle_rule.py +++ b/trestlebot/transformers/trestle_rule.py @@ -14,17 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. -"""Trestle Rule Dataclass and base transformer.""" +"""Trestle Rule Dataclass.""" -from abc import abstractmethod -from dataclasses import dataclass from typing import Any, Dict, List, Optional -from trestle.transforms.transformer_factory import TransformerBase +from pydantic import BaseModel -@dataclass -class Parameter: +class Parameter(BaseModel): """Parameter dataclass.""" name: str @@ -33,15 +30,13 @@ class Parameter: default_value: str -@dataclass -class Control: +class Control(BaseModel): """Control dataclass.""" id: str -@dataclass -class Profile: +class Profile(BaseModel): """Profile dataclass.""" description: str @@ -49,8 +44,7 @@ class Profile: include_controls: List[Control] -@dataclass -class ComponentInfo: +class ComponentInfo(BaseModel): """ComponentInfo dataclass.""" name: str @@ -58,8 +52,7 @@ class ComponentInfo: description: str -@dataclass -class TrestleRule: +class TrestleRule(BaseModel): """TrestleRule dataclass.""" name: str @@ -67,15 +60,3 @@ class TrestleRule: component: ComponentInfo parameter: Optional[Parameter] profile: Profile - - -class RulesTransformer(TransformerBase): - """Abstract interface for transformers for rules""" - - @abstractmethod - def transform(self, blob: str) -> TrestleRule: - """Transform rule data.""" - - -class RulesTransformerException(Exception): - """An error during transformation of a rule""" diff --git a/trestlebot/transformers/yaml_transformer.py b/trestlebot/transformers/yaml_transformer.py new file mode 100644 index 00000000..b24e227f --- /dev/null +++ b/trestlebot/transformers/yaml_transformer.py @@ -0,0 +1,119 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""YAML to CSV transformer for rule authoring.""" +import logging +from io import StringIO +from typing import Any, Dict + +from pydantic import ValidationError +from ruamel.yaml import YAML + +from trestlebot import const +from trestlebot.transformers.base_transformer import ( + RulesTransformer, + RulesTransformerException, +) +from trestlebot.transformers.trestle_rule import ( + ComponentInfo, + Parameter, + Profile, + TrestleRule, +) + + +logger = logging.getLogger(__name__) + + +class RulesYAMLTransformer(RulesTransformer): + """Interface for YAML transformer to Rules model.""" + + def __init__(self) -> None: + """Initialize.""" + super().__init__() + + def transform_to_rule(self, blob: str) -> TrestleRule: + """Rules YAML data into a row of CSV.""" + trestle_rule: TrestleRule = self._ingest_yaml(blob) + return trestle_rule + + def transform_from_rule(self, trestle: TrestleRule) -> str: + """Rules YAML data into a row of CSV.""" + return self._write_yaml(trestle) + + @staticmethod + def _write_yaml(rule: TrestleRule) -> str: + """Write the YAML to a string.""" + yaml = YAML() + yaml.default_flow_style = False + yaml_data: Dict[str, Any] = { + const.RULE_INFO_TAG: { + const.NAME: rule.name, + const.DESCRIPTION: rule.description, + const.PROFILE: rule.profile.dict(by_alias=True, exclude_unset=True), + }, + const.COMPONENT_INFO_TAG: rule.component.dict( + by_alias=True, exclude_unset=True + ), + } + + if rule.parameter is not None: + yaml_data[const.RULE_INFO_TAG][const.PARAMETER] = rule.parameter.dict( + by_alias=True, exclude_unset=True + ) + + yaml_stream = StringIO() + yaml.dump(yaml_data, yaml_stream) + return yaml_stream.getvalue() + + @staticmethod + def _ingest_yaml(blob: str) -> TrestleRule: + """Ingest the YAML blob into a TrestleData object.""" + try: + yaml = YAML(typ="safe") + yaml_data: Dict[str, Any] = yaml.load(blob) + + rule_info_data = yaml_data[const.RULE_INFO_TAG] + + profile_data = rule_info_data[const.PROFILE] + profile_info_instance: Profile = Profile(**profile_data) + + component_info_data = yaml_data[const.COMPONENT_INFO_TAG] + component_info_instance: ComponentInfo = ComponentInfo( + **component_info_data + ) + + rule_info_instance: TrestleRule = TrestleRule( + name=rule_info_data[const.NAME], + description=rule_info_data[const.DESCRIPTION], + component=component_info_instance, + parameter=None, + profile=profile_info_instance, + ) + + if const.PARAMETER in rule_info_data: + parameter_data = rule_info_data[const.PARAMETER] + parameter_instance: Parameter = Parameter(**parameter_data) + rule_info_instance.parameter = parameter_instance + + except KeyError as e: + raise RulesTransformerException(f"Missing key in YAML file: {e}") + except ValidationError as e: + raise RulesTransformerException(f"Invalid YAML file: {e}") + except Exception as e: + raise RuntimeError(e) + + return rule_info_instance From 39da77d16ff8aa91815f7984540790d72b41efb3 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Tue, 3 Oct 2023 19:22:57 -0400 Subject: [PATCH 2/6] docs: updates comment Signed-off-by: Jennifer Power --- trestlebot/transformers/csv_to_yaml.py | 1 + trestlebot/transformers/csv_transformer.py | 2 ++ trestlebot/transformers/trestle_rule.py | 2 +- trestlebot/transformers/yaml_transformer.py | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/trestlebot/transformers/csv_to_yaml.py b/trestlebot/transformers/csv_to_yaml.py index d1ab2ce0..8cef9941 100644 --- a/trestlebot/transformers/csv_to_yaml.py +++ b/trestlebot/transformers/csv_to_yaml.py @@ -13,6 +13,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +"""CSV to YAML converter for rule authoring.""" import csv import pathlib diff --git a/trestlebot/transformers/csv_transformer.py b/trestlebot/transformers/csv_transformer.py index 5c39c550..90cc40ee 100644 --- a/trestlebot/transformers/csv_transformer.py +++ b/trestlebot/transformers/csv_transformer.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +"""CSV Tranformer for rule authoring.""" + import csv import json import logging diff --git a/trestlebot/transformers/trestle_rule.py b/trestlebot/transformers/trestle_rule.py index 87630ae9..6a52de37 100644 --- a/trestlebot/transformers/trestle_rule.py +++ b/trestlebot/transformers/trestle_rule.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Trestle Rule Dataclass.""" +"""Trestle Rule class with pydantic.""" from typing import Any, Dict, List, Optional diff --git a/trestlebot/transformers/yaml_transformer.py b/trestlebot/transformers/yaml_transformer.py index b24e227f..10f2f80c 100644 --- a/trestlebot/transformers/yaml_transformer.py +++ b/trestlebot/transformers/yaml_transformer.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""YAML to CSV transformer for rule authoring.""" +"""YAML transformer for rule authoring.""" import logging from io import StringIO from typing import Any, Dict From 8cd41d5763df1660282c6c7404ca808715a6913e Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Tue, 3 Oct 2023 20:38:14 -0400 Subject: [PATCH 3/6] docs: improves comment and variable naming Signed-off-by: Jennifer Power --- trestlebot/transformers/__init__.py | 4 +++ trestlebot/transformers/csv_transformer.py | 39 ++++++++++++--------- trestlebot/transformers/yaml_transformer.py | 16 +++++---- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/trestlebot/transformers/__init__.py b/trestlebot/transformers/__init__.py index 034efddd..d160aa73 100644 --- a/trestlebot/transformers/__init__.py +++ b/trestlebot/transformers/__init__.py @@ -17,4 +17,8 @@ Trestlebot transformers. Custom transformers using trestle libs for trestlebot. + +The RulesTransformer class is the base class for all transformers pertaining +to TrestleRule objects. This allows us to have a common interface for all formats +that express rules. """ diff --git a/trestlebot/transformers/csv_transformer.py b/trestlebot/transformers/csv_transformer.py index 90cc40ee..5ff8a526 100644 --- a/trestlebot/transformers/csv_transformer.py +++ b/trestlebot/transformers/csv_transformer.py @@ -56,7 +56,12 @@ class RulesCSVTransformer(RulesTransformer): - """Interface for CSV transformer to Rules model.""" + """ + Interface for CSV transformer to Rules model. + + Notes: This will transform individual rows of CSV to and from a + Trestle object with row compliance with the Trestle CSV requirements. + """ def __init__(self) -> None: """Initialize.""" @@ -77,6 +82,22 @@ def transform_to_rule(self, row: Dict[str, str]) -> TrestleRule: profile=profile, ) + def transform_from_rule(self, rule: TrestleRule) -> Dict[str, str]: + """Transforms TrestleRule into a row of CSV.""" + rule_dict: Dict[str, str] = { + RULE_ID: rule.name, + RULE_DESCRIPTION: rule.description, + NAMESPACE: TRESTLE_GENERIC_NS, + } + merged_dict = { + **rule_dict, + **self._add_profile(rule.profile), + **self._add_component_info(rule.component), + } + if rule.parameter is not None: + merged_dict.update(self._add_parameter(rule.parameter)) + return merged_dict + def _extract_rule_info(self, row: Dict[str, str]) -> Dict[str, str]: """Extract rule information from a CSV row.""" return { @@ -117,22 +138,6 @@ def _extract_component_info(self, row: Dict[str, str]) -> ComponentInfo: description=row.get(csv_to_oscal_cd.COMPONENT_DESCRIPTION, ""), ) - def transform_from_rule(self, rule: TrestleRule) -> Dict[str, str]: - """Rules YAML data into a row of CSV.""" - rule_dict: Dict[str, str] = { - RULE_ID: rule.name, - RULE_DESCRIPTION: rule.description, - NAMESPACE: TRESTLE_GENERIC_NS, - } - merged_dict = { - **rule_dict, - **self._add_profile(rule.profile), - **self._add_component_info(rule.component), - } - if rule.parameter is not None: - merged_dict.update(self._add_parameter(rule.parameter)) - return merged_dict - def _add_profile(self, profile: Profile) -> Dict[str, str]: """Add a profile to the CSV Row.""" controls_list: List[str] = [control.id for control in profile.include_controls] diff --git a/trestlebot/transformers/yaml_transformer.py b/trestlebot/transformers/yaml_transformer.py index 10f2f80c..52351f7e 100644 --- a/trestlebot/transformers/yaml_transformer.py +++ b/trestlebot/transformers/yaml_transformer.py @@ -46,20 +46,21 @@ def __init__(self) -> None: super().__init__() def transform_to_rule(self, blob: str) -> TrestleRule: - """Rules YAML data into a row of CSV.""" + """Transform YAML data into a TrestleRule object.""" trestle_rule: TrestleRule = self._ingest_yaml(blob) return trestle_rule def transform_from_rule(self, trestle: TrestleRule) -> str: - """Rules YAML data into a row of CSV.""" + """Transform TrestleRule object into YAML data.""" return self._write_yaml(trestle) @staticmethod def _write_yaml(rule: TrestleRule) -> str: """Write the YAML to a string.""" - yaml = YAML() - yaml.default_flow_style = False - yaml_data: Dict[str, Any] = { + yaml_obj = YAML() + yaml_obj.default_flow_style = False + + rule_info: Dict[str, Any] = { const.RULE_INFO_TAG: { const.NAME: rule.name, const.DESCRIPTION: rule.description, @@ -71,12 +72,13 @@ def _write_yaml(rule: TrestleRule) -> str: } if rule.parameter is not None: - yaml_data[const.RULE_INFO_TAG][const.PARAMETER] = rule.parameter.dict( + rule_info[const.RULE_INFO_TAG][const.PARAMETER] = rule.parameter.dict( by_alias=True, exclude_unset=True ) yaml_stream = StringIO() - yaml.dump(yaml_data, yaml_stream) + yaml_obj.dump(rule_info, yaml_stream) + return yaml_stream.getvalue() @staticmethod From 693620d90d86ce88c8d4a77193e5af97714a913f Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Wed, 4 Oct 2023 07:11:03 -0400 Subject: [PATCH 4/6] refactor: reduce extra code in RulesYAMLTransformer for simplicity Signed-off-by: Jennifer Power --- trestlebot/transformers/csv_to_yaml.py | 7 ++- trestlebot/transformers/yaml_transformer.py | 63 +++++++++------------ 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/trestlebot/transformers/csv_to_yaml.py b/trestlebot/transformers/csv_to_yaml.py index 8cef9941..7c005e32 100644 --- a/trestlebot/transformers/csv_to_yaml.py +++ b/trestlebot/transformers/csv_to_yaml.py @@ -55,7 +55,10 @@ def write_to_yaml(self, filepath: pathlib.Path) -> None: try: with open(filepath, "w") as yaml_file: yaml.dump( - [self._yaml_transformer._write_yaml(rule) for rule in self._rules], + [ + self._yaml_transformer.transform_from_rule(rule) + for rule in self._rules + ], yaml_file, ) except Exception as e: @@ -78,7 +81,7 @@ def write_default_trestle_rule_keys(self, filepath: pathlib.Path) -> None: include_controls=[Control(id="example")], ), ) - yaml_blob = self._yaml_transformer._write_yaml(test_rule) + yaml_blob = self._yaml_transformer.transform_from_rule(test_rule) with open(filepath, "w") as yaml_file: yaml_file.write(yaml_blob) except Exception as e: diff --git a/trestlebot/transformers/yaml_transformer.py b/trestlebot/transformers/yaml_transformer.py index 52351f7e..254759e3 100644 --- a/trestlebot/transformers/yaml_transformer.py +++ b/trestlebot/transformers/yaml_transformer.py @@ -47,43 +47,6 @@ def __init__(self) -> None: def transform_to_rule(self, blob: str) -> TrestleRule: """Transform YAML data into a TrestleRule object.""" - trestle_rule: TrestleRule = self._ingest_yaml(blob) - return trestle_rule - - def transform_from_rule(self, trestle: TrestleRule) -> str: - """Transform TrestleRule object into YAML data.""" - return self._write_yaml(trestle) - - @staticmethod - def _write_yaml(rule: TrestleRule) -> str: - """Write the YAML to a string.""" - yaml_obj = YAML() - yaml_obj.default_flow_style = False - - rule_info: Dict[str, Any] = { - const.RULE_INFO_TAG: { - const.NAME: rule.name, - const.DESCRIPTION: rule.description, - const.PROFILE: rule.profile.dict(by_alias=True, exclude_unset=True), - }, - const.COMPONENT_INFO_TAG: rule.component.dict( - by_alias=True, exclude_unset=True - ), - } - - if rule.parameter is not None: - rule_info[const.RULE_INFO_TAG][const.PARAMETER] = rule.parameter.dict( - by_alias=True, exclude_unset=True - ) - - yaml_stream = StringIO() - yaml_obj.dump(rule_info, yaml_stream) - - return yaml_stream.getvalue() - - @staticmethod - def _ingest_yaml(blob: str) -> TrestleRule: - """Ingest the YAML blob into a TrestleData object.""" try: yaml = YAML(typ="safe") yaml_data: Dict[str, Any] = yaml.load(blob) @@ -119,3 +82,29 @@ def _ingest_yaml(blob: str) -> TrestleRule: raise RuntimeError(e) return rule_info_instance + + def transform_from_rule(self, rule: TrestleRule) -> str: + """Transform TrestleRule object into YAML data.""" + yaml_obj = YAML() + yaml_obj.default_flow_style = False + + rule_info: Dict[str, Any] = { + const.RULE_INFO_TAG: { + const.NAME: rule.name, + const.DESCRIPTION: rule.description, + const.PROFILE: rule.profile.dict(by_alias=True, exclude_unset=True), + }, + const.COMPONENT_INFO_TAG: rule.component.dict( + by_alias=True, exclude_unset=True + ), + } + + if rule.parameter is not None: + rule_info[const.RULE_INFO_TAG][const.PARAMETER] = rule.parameter.dict( + by_alias=True, exclude_unset=True + ) + + yaml_stream = StringIO() + yaml_obj.dump(rule_info, yaml_stream) + + return yaml_stream.getvalue() From 78df3d8ea9ad9270b9472581dc255686216e2e89 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Wed, 4 Oct 2023 11:33:09 -0400 Subject: [PATCH 5/6] feat: adds Field aliases to remove underscores from YAML Signed-off-by: Jennifer Power --- tests/data/yaml/test_complete_rule.yaml | 6 +++--- tests/data/yaml/test_complete_rule_no_params.yaml | 2 +- tests/data/yaml/test_incomplete_rule.yaml | 6 +++--- tests/data/yaml/test_invalid_rule.yaml | 6 +++--- tests/trestlebot/transformers/test_csv_to_yaml.py | 3 +++ trestlebot/const.py | 4 ---- trestlebot/transformers/trestle_rule.py | 14 ++++++++++---- trestlebot/transformers/yaml_transformer.py | 8 ++++---- 8 files changed, 27 insertions(+), 22 deletions(-) diff --git a/tests/data/yaml/test_complete_rule.yaml b/tests/data/yaml/test_complete_rule.yaml index 9f41c19f..2a37605a 100644 --- a/tests/data/yaml/test_complete_rule.yaml +++ b/tests/data/yaml/test_complete_rule.yaml @@ -4,12 +4,12 @@ x-trestle-rule-info: parameter: name: prm_1 description: prm_1 description - alternative_values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} - default_value: '5%' + alternative-values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} + default-value: '5%' profile: description: Simple NIST Profile href: profiles/simplified_nist_profile/profile.json - include_controls: + include-controls: - id: ac-1 x-trestle-component-info: name: Component 1 diff --git a/tests/data/yaml/test_complete_rule_no_params.yaml b/tests/data/yaml/test_complete_rule_no_params.yaml index 0e005f3c..f95d72f7 100644 --- a/tests/data/yaml/test_complete_rule_no_params.yaml +++ b/tests/data/yaml/test_complete_rule_no_params.yaml @@ -4,7 +4,7 @@ x-trestle-rule-info: profile: description: Simple NIST Profile href: profiles/simplified_nist_profile/profile.json - include_controls: + include-controls: - id: ac-1 x-trestle-component-info: name: Component 1 diff --git a/tests/data/yaml/test_incomplete_rule.yaml b/tests/data/yaml/test_incomplete_rule.yaml index ad368ac9..e8e2ec33 100644 --- a/tests/data/yaml/test_incomplete_rule.yaml +++ b/tests/data/yaml/test_incomplete_rule.yaml @@ -3,12 +3,12 @@ x-trestle-rule-info: parameter: name: prm_1 description: prm_1 description - alternative_values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} - default_value: '5%' + alternative-values: {'default': '5%', '5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} + default-value: '5%' profile: description: Simple NIST Profile href: profiles/simplified_nist_profile/profile.json - include_controls: + include-controls: - id: ac-2 x-trestle-component-info: name: Component 1 diff --git a/tests/data/yaml/test_invalid_rule.yaml b/tests/data/yaml/test_invalid_rule.yaml index cf6cb98e..6062e47a 100644 --- a/tests/data/yaml/test_invalid_rule.yaml +++ b/tests/data/yaml/test_invalid_rule.yaml @@ -4,12 +4,12 @@ x-trestle-rule-info: parameter: name: prm_1 description: prm_1 description - alternative_values: "invalid" - default_value: true + alternative-values: "invalid" + default-value: true profile: description: Simple NIST Profile href: profiles/simplified_nist_profile/profile.json - include_controls: + include-controls: - id: ac-2 x-trestle-component-info: name: Component 1 diff --git a/tests/trestlebot/transformers/test_csv_to_yaml.py b/tests/trestlebot/transformers/test_csv_to_yaml.py index d85f177e..17905c97 100644 --- a/tests/trestlebot/transformers/test_csv_to_yaml.py +++ b/tests/trestlebot/transformers/test_csv_to_yaml.py @@ -80,6 +80,9 @@ def test_default_test_trestle_rule_keys( ) -> None: yaml_file = pathlib.Path(tmp_trestle_dir) / "test.yaml" setup_yaml_builder.write_default_trestle_rule_keys(yaml_file) + + # Check that the YAML file written is valid and integrates with the rule + # YAML transformer transformer = RulesYAMLTransformer() rule = transformer.transform_to_rule(yaml_file.read_text()) diff --git a/trestlebot/const.py b/trestlebot/const.py index 5d905e80..7d96d2ae 100644 --- a/trestlebot/const.py +++ b/trestlebot/const.py @@ -36,10 +36,6 @@ DESCRIPTION = "description" PARAMETER = "parameter" PROFILE = "profile" -HREF = "href" -INCLUDE_CONTROLS = "include-controls" -DEFAULT_VALUE = "default-value" -ALTERNATIVE_VALUES = "alternative-values" COMPONENT_YAML = "component.yaml" COMPONENT_INFO_TAG = trestle_const.TRESTLE_TAG + "component-info" diff --git a/trestlebot/transformers/trestle_rule.py b/trestlebot/transformers/trestle_rule.py index 6a52de37..23be279a 100644 --- a/trestlebot/transformers/trestle_rule.py +++ b/trestlebot/transformers/trestle_rule.py @@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field class Parameter(BaseModel): @@ -26,8 +26,11 @@ class Parameter(BaseModel): name: str description: str - alternative_values: Dict[str, Any] - default_value: str + alternative_values: Dict[str, Any] = Field(..., alias="alternative-values") + default_value: str = Field(..., alias="default-value") + + class Config: + allow_population_by_field_name = True class Control(BaseModel): @@ -41,7 +44,10 @@ class Profile(BaseModel): description: str href: str - include_controls: List[Control] + include_controls: List[Control] = Field(..., alias="include-controls") + + class Config: + allow_population_by_field_name = True class ComponentInfo(BaseModel): diff --git a/trestlebot/transformers/yaml_transformer.py b/trestlebot/transformers/yaml_transformer.py index 254759e3..d967e33b 100644 --- a/trestlebot/transformers/yaml_transformer.py +++ b/trestlebot/transformers/yaml_transformer.py @@ -54,11 +54,11 @@ def transform_to_rule(self, blob: str) -> TrestleRule: rule_info_data = yaml_data[const.RULE_INFO_TAG] profile_data = rule_info_data[const.PROFILE] - profile_info_instance: Profile = Profile(**profile_data) + profile_info_instance: Profile = Profile.parse_obj(profile_data) component_info_data = yaml_data[const.COMPONENT_INFO_TAG] - component_info_instance: ComponentInfo = ComponentInfo( - **component_info_data + component_info_instance: ComponentInfo = ComponentInfo.parse_obj( + component_info_data ) rule_info_instance: TrestleRule = TrestleRule( @@ -71,7 +71,7 @@ def transform_to_rule(self, blob: str) -> TrestleRule: if const.PARAMETER in rule_info_data: parameter_data = rule_info_data[const.PARAMETER] - parameter_instance: Parameter = Parameter(**parameter_data) + parameter_instance: Parameter = Parameter.parse_obj(parameter_data) rule_info_instance.parameter = parameter_instance except KeyError as e: From 56c5b52e8c28354ecedf2d1b37aa279ad731653c Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Tue, 3 Oct 2023 21:11:22 -0400 Subject: [PATCH 6/6] refactor: breaks transformer into ToRuleTransformer and FromRuleTransformer To align with upstream trestle, the transformers are broken down into single responsibilties and importing the Transformer Base. Signed-off-by: Jennifer Power --- tests/conftest.py | 106 +++++++++++++++++- tests/data/yaml/test_rule_invalid_params.yaml | 17 +++ .../tasks/test_rule_transform_task.py | 10 +- .../transformers/test_csv_to_yaml.py | 14 ++- .../transformers/test_csv_transformer.py | 50 +++------ .../transformers/test_yaml_transformer.py | 32 ++++-- trestlebot/const.py | 3 + trestlebot/tasks/rule_transform_task.py | 12 +- trestlebot/transformers/base_transformer.py | 16 ++- trestlebot/transformers/csv_to_yaml.py | 36 +++--- trestlebot/transformers/csv_transformer.py | 58 ++++++---- trestlebot/transformers/yaml_transformer.py | 50 +++++++-- 12 files changed, 284 insertions(+), 120 deletions(-) create mode 100644 tests/data/yaml/test_rule_invalid_params.yaml diff --git a/tests/conftest.py b/tests/conftest.py index eea79001..8383feaf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,13 +20,22 @@ import os import pathlib from tempfile import TemporaryDirectory -from typing import Generator, Tuple, TypeVar +from typing import Any, Dict, Generator, Tuple, TypeVar import pytest from git.repo import Repo from trestle.common.err import TrestleError from trestle.core.commands.init import InitCmd +from trestlebot import const +from trestlebot.transformers.trestle_rule import ( + ComponentInfo, + Control, + Parameter, + Profile, + TrestleRule, +) + T = TypeVar("T") @@ -74,3 +83,98 @@ def tmp_trestle_dir() -> YieldFixture[str]: f"Initialization failed for temporary trestle directory: {e}." ) yield tmpdir + + +@pytest.fixture(scope="function") +def valid_rule_data() -> Dict[str, Any]: + return { + const.RULE_INFO_TAG: { + const.NAME: "example_rule_1", + const.DESCRIPTION: "My rule description for example rule 1", + const.PROFILE: { + const.DESCRIPTION: "Simple NIST Profile", + const.HREF: "profiles/simplified_nist_profile/profile.json", + }, + const.PARAMETER: { + const.NAME: "prm_1", + const.DESCRIPTION: "prm_1 description", + const.ALTERNATIVE_VALUES: { + "default": "5%", + "5pc": "5%", + "10pc": "10%", + "15pc": "15%", + "20pc": "20%", + }, + const.DEFAULT_VALUE: "5%", + }, + } + } + + +@pytest.fixture(scope="function") +def invalid_param_rule_data() -> Dict[str, Any]: + return { + const.RULE_INFO_TAG: { + const.NAME: "example_rule_1", + const.DESCRIPTION: "My rule description for example rule 1", + const.PROFILE: { + const.DESCRIPTION: "Simple NIST Profile", + const.HREF: "profiles/simplified_nist_profile/profile.json", + }, + const.PARAMETER: { + const.NAME: "prm_1", + const.DESCRIPTION: "prm_1 description", + const.ALTERNATIVE_VALUES: { + "5pc": "5%", + "10pc": "10%", + "15pc": "15%", + "20pc": "20%", + }, + const.DEFAULT_VALUE: "5%", + }, + } + } + + +@pytest.fixture(scope="function") +def missing_key_rule_data() -> Dict[str, Any]: + return { + const.RULE_INFO_TAG: { + const.DESCRIPTION: "My rule description for example rule 1", + const.PROFILE: { + const.DESCRIPTION: "Simple NIST Profile", + const.HREF: "profiles/simplified_nist_profile/profile.json", + }, + const.PARAMETER: { + const.NAME: "prm_1", + const.DESCRIPTION: "prm_1 description", + const.ALTERNATIVE_VALUES: { + "default": "5%", + "5pc": "5%", + "10pc": "10%", + "15pc": "15%", + "20pc": "20%", + }, + const.DEFAULT_VALUE: "5%", + }, + } + } + + +@pytest.fixture(scope="function") +def test_rule() -> TrestleRule: + test_trestle_rule: TrestleRule = TrestleRule( + name="test", + description="test", + component=ComponentInfo(name="test_comp", type="test", description="test"), + parameter=Parameter( + name="test", + description="test", + alternative_values={}, + default_value="test", + ), + profile=Profile( + description="test", href="test", include_controls=[Control(id="ac-1")] + ), + ) + return test_trestle_rule diff --git a/tests/data/yaml/test_rule_invalid_params.yaml b/tests/data/yaml/test_rule_invalid_params.yaml new file mode 100644 index 00000000..3b4acdcd --- /dev/null +++ b/tests/data/yaml/test_rule_invalid_params.yaml @@ -0,0 +1,17 @@ +x-trestle-rule-info: + name: example_rule_1 + description: My rule description for example rule 1 + parameter: + name: prm_1 + description: prm_1 description + alternative-values: {'5pc': '5%', '10pc': '10%', '15pc': '15%', '20pc': '20%'} + default-value: '5%' + profile: + description: Simple NIST Profile + href: profiles/simplified_nist_profile/profile.json + include-controls: + - id: ac-2 +x-trestle-component-info: + name: Component 1 + description: Component 1 description + type: service diff --git a/tests/trestlebot/tasks/test_rule_transform_task.py b/tests/trestlebot/tasks/test_rule_transform_task.py index 37869144..9fb61200 100644 --- a/tests/trestlebot/tasks/test_rule_transform_task.py +++ b/tests/trestlebot/tasks/test_rule_transform_task.py @@ -28,7 +28,7 @@ from tests.testutils import setup_rules_view from trestlebot.tasks.base_task import TaskException from trestlebot.tasks.rule_transform_task import RuleTransformTask -from trestlebot.transformers.yaml_transformer import RulesYAMLTransformer +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer test_comp = "test_comp" @@ -39,7 +39,7 @@ def test_rule_transform_task(tmp_trestle_dir: str) -> None: """Test rule transform task.""" trestle_root = pathlib.Path(tmp_trestle_dir) setup_rules_view(trestle_root, test_comp, test_rules_dir) - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() rule_transform_task = RuleTransformTask( tmp_trestle_dir, test_rules_dir, transformer ) @@ -70,7 +70,7 @@ def test_rule_transform_task_with_no_rules(tmp_trestle_dir: str) -> None: """Test rule transform task with no rules.""" trestle_root = pathlib.Path(tmp_trestle_dir) setup_rules_view(trestle_root, test_comp, test_rules_dir, skip_rules=True) - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() rule_transform_task = RuleTransformTask( tmp_trestle_dir, test_rules_dir, transformer ) @@ -85,7 +85,7 @@ def test_rule_transform_task_with_invalid_rule(tmp_trestle_dir: str) -> None: """Test rule transform task with invalid rule.""" trestle_root = pathlib.Path(tmp_trestle_dir) setup_rules_view(trestle_root, test_comp, test_rules_dir, incomplete_rule=True) - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() rule_transform_task = RuleTransformTask( tmp_trestle_dir, test_rules_dir, transformer ) @@ -100,7 +100,7 @@ def test_rule_transform_task_with_skip(tmp_trestle_dir: str) -> None: """Test rule transform task with skip.""" trestle_root = pathlib.Path(tmp_trestle_dir) setup_rules_view(trestle_root, test_comp, test_rules_dir) - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() rule_transform_task = RuleTransformTask( tmp_trestle_dir, test_rules_dir, transformer, skip_model_list=[test_comp] ) diff --git a/tests/trestlebot/transformers/test_csv_to_yaml.py b/tests/trestlebot/transformers/test_csv_to_yaml.py index 17905c97..42a600a0 100644 --- a/tests/trestlebot/transformers/test_csv_to_yaml.py +++ b/tests/trestlebot/transformers/test_csv_to_yaml.py @@ -18,10 +18,10 @@ import pathlib import pytest -import ruamel.yaml as yaml +from ruamel.yaml import YAML from trestlebot.transformers.csv_to_yaml import YAMLBuilder -from trestlebot.transformers.yaml_transformer import RulesYAMLTransformer +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer @pytest.fixture(scope="function") @@ -70,9 +70,11 @@ def test_write_to_yaml(setup_yaml_builder: YAMLBuilder, tmp_trestle_dir: str) -> write_sample_csv(csv_file) setup_yaml_builder.read_from_csv(csv_file) setup_yaml_builder.write_to_yaml(yaml_file) + yaml = YAML(typ="safe") with open(yaml_file, "r") as f: - data = yaml.safe_load(f) - assert len(data) == 1 + data = yaml.load(f) + # The file will contain a separate YAML document for each rule + assert len(data) == 2 def test_default_test_trestle_rule_keys( @@ -83,8 +85,8 @@ def test_default_test_trestle_rule_keys( # Check that the YAML file written is valid and integrates with the rule # YAML transformer - transformer = RulesYAMLTransformer() - rule = transformer.transform_to_rule(yaml_file.read_text()) + transformer = ToRulesYAMLTransformer() + rule = transformer.transform(yaml_file.read_text()) assert rule.name == "example rule" assert rule.description == "example description" diff --git a/tests/trestlebot/transformers/test_csv_transformer.py b/tests/trestlebot/transformers/test_csv_transformer.py index b3d595d9..6dc61bb4 100644 --- a/tests/trestlebot/transformers/test_csv_transformer.py +++ b/tests/trestlebot/transformers/test_csv_transformer.py @@ -23,51 +23,29 @@ import pytest from trestlebot.transformers.csv_transformer import CSVBuilder -from trestlebot.transformers.trestle_rule import ( - ComponentInfo, - Control, - Parameter, - Profile, - TrestleRule, -) +from trestlebot.transformers.trestle_rule import TrestleRule -test_comp = "my comp" - - -def test_csv_builder(tmp_trestle_dir: str) -> None: +def test_csv_builder(test_rule: TrestleRule, tmp_trestle_dir: str) -> None: """Test CSV builder on a happy path""" - test_trestle_rule: TrestleRule = TrestleRule( - name="test", - description="test", - component=ComponentInfo(name=test_comp, type="test", description="test"), - parameter=Parameter( - name="test", - description="test", - alternative_values={}, - default_value="test", - ), - profile=Profile( - description="test", href="test", include_controls=[Control(id="ac-1")] - ), - ) + csv_builder = CSVBuilder() - csv_builder.add_row(test_trestle_rule) + csv_builder.add_row(test_rule) assert len(csv_builder._rows) == 1 row = csv_builder._rows[0] - assert row["Rule_Id"] == test_trestle_rule.name - assert row["Rule_Description"] == test_trestle_rule.description - assert row["Component_Title"] == test_trestle_rule.component.name - assert row["Component_Type"] == test_trestle_rule.component.type - assert row["Component_Description"] == test_trestle_rule.component.description + assert row["Rule_Id"] == test_rule.name + assert row["Rule_Description"] == test_rule.description + assert row["Component_Title"] == test_rule.component.name + assert row["Component_Type"] == test_rule.component.type + assert row["Component_Description"] == test_rule.component.description assert row["Control_Id_List"] == "ac-1" - assert row["Parameter_Id"] == test_trestle_rule.parameter.name # type: ignore - assert row["Parameter_Description"] == test_trestle_rule.parameter.description # type: ignore + assert row["Parameter_Id"] == test_rule.parameter.name # type: ignore + assert row["Parameter_Description"] == test_rule.parameter.description # type: ignore assert row["Parameter_Value_Alternatives"] == "{}" - assert row["Parameter_Value_Default"] == test_trestle_rule.parameter.default_value # type: ignore - assert row["Profile_Description"] == test_trestle_rule.profile.description - assert row["Profile_Source"] == test_trestle_rule.profile.href + assert row["Parameter_Value_Default"] == test_rule.parameter.default_value # type: ignore + assert row["Profile_Description"] == test_rule.profile.description + assert row["Profile_Source"] == test_rule.profile.href trestle_root = pathlib.Path(tmp_trestle_dir) tmp_csv_path = trestle_root.joinpath("test.csv") diff --git a/tests/trestlebot/transformers/test_yaml_transformer.py b/tests/trestlebot/transformers/test_yaml_transformer.py index c393eae5..55bbb14d 100644 --- a/tests/trestlebot/transformers/test_yaml_transformer.py +++ b/tests/trestlebot/transformers/test_yaml_transformer.py @@ -20,7 +20,11 @@ from tests.testutils import YAML_TEST_DATA_PATH from trestlebot.transformers.base_transformer import RulesTransformerException -from trestlebot.transformers.yaml_transformer import RulesYAMLTransformer +from trestlebot.transformers.trestle_rule import TrestleRule +from trestlebot.transformers.yaml_transformer import ( + FromRulesYAMLTransformer, + ToRulesYAMLTransformer, +) def test_rule_transformer() -> None: @@ -32,8 +36,8 @@ def test_rule_transformer() -> None: rule_file_info = rule_file.read() rule_file.close() - transformer = RulesYAMLTransformer() - rule = transformer.transform_to_rule(rule_file_info) + transformer = ToRulesYAMLTransformer() + rule = transformer.transform(rule_file_info) assert rule.name == "example_rule_1" assert rule.description == "My rule description for example rule 1" @@ -59,13 +63,13 @@ def test_rules_transform_with_incomplete_rule() -> None: """Test rules transform with incomplete rule.""" # Generate test json string test_string = '{"test_json": "test"}' - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() with pytest.raises( RulesTransformerException, match="Missing key in YAML file: 'x-trestle-rule-info'", ): - transformer.transform_to_rule(test_string) + transformer.transform(test_string) def test_rules_transform_with_invalid_rule() -> None: @@ -76,9 +80,23 @@ def test_rules_transform_with_invalid_rule() -> None: rule_file = open(rule_path, "r") rule_file_info = rule_file.read() rule_file.close() - transformer = RulesYAMLTransformer() + transformer = ToRulesYAMLTransformer() with pytest.raises( RulesTransformerException, match="Invalid YAML file: 1 validation error .*" ): - transformer.transform_to_rule(rule_file_info) + transformer.transform(rule_file_info) + + +def test_read_write_integration(test_rule: TrestleRule) -> None: + """Test read/write integration.""" + from_rules_transformer = FromRulesYAMLTransformer() + to_rules_transformer = ToRulesYAMLTransformer() + + yaml_data = from_rules_transformer.transform(test_rule) + + blob = yaml_data.getvalue() + + read_rule = to_rules_transformer.transform(blob) + + assert read_rule == test_rule diff --git a/trestlebot/const.py b/trestlebot/const.py index 7d96d2ae..f7ad568a 100644 --- a/trestlebot/const.py +++ b/trestlebot/const.py @@ -36,6 +36,9 @@ DESCRIPTION = "description" PARAMETER = "parameter" PROFILE = "profile" +HREF = "href" +ALTERNATIVE_VALUES = "alternative-values" +DEFAULT_VALUE = "default-value" COMPONENT_YAML = "component.yaml" COMPONENT_INFO_TAG = trestle_const.TRESTLE_TAG + "component-info" diff --git a/trestlebot/tasks/rule_transform_task.py b/trestlebot/tasks/rule_transform_task.py index 423f915a..7944673e 100644 --- a/trestlebot/tasks/rule_transform_task.py +++ b/trestlebot/tasks/rule_transform_task.py @@ -28,11 +28,9 @@ import trestlebot.const as const from trestlebot.tasks.base_task import TaskBase, TaskException -from trestlebot.transformers.base_transformer import ( - RulesTransformer, - RulesTransformerException, -) +from trestlebot.transformers.base_transformer import RulesTransformerException from trestlebot.transformers.csv_transformer import CSVBuilder +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer logger = logging.getLogger(__name__) @@ -47,7 +45,7 @@ def __init__( self, working_dir: str, rules_view_dir: str, - rule_transformer: RulesTransformer, + rule_transformer: ToRulesYAMLTransformer, skip_model_list: List[str] = [], ) -> None: """ @@ -67,7 +65,7 @@ def __init__( """ self._rule_view_dir = rules_view_dir - self._rule_transformer: RulesTransformer = rule_transformer + self._rule_transformer: ToRulesYAMLTransformer = rule_transformer super().__init__(working_dir, skip_model_list) def execute(self) -> int: @@ -101,7 +99,7 @@ def _transform_components(self, component_definition_path: pathlib.Path) -> None rule_stream = rule_path.read_text() try: - rule = self._rule_transformer.transform_to_rule(rule_stream) + rule = self._rule_transformer.transform(rule_stream) csv_builder.add_row(rule) except RulesTransformerException as e: raise TaskException( diff --git a/trestlebot/transformers/base_transformer.py b/trestlebot/transformers/base_transformer.py index 3d7538b0..bac21150 100644 --- a/trestlebot/transformers/base_transformer.py +++ b/trestlebot/transformers/base_transformer.py @@ -16,21 +16,27 @@ """Base transformer for rules.""" -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import Any +from trestle.transforms.transformer_factory import TransformerBase + from trestlebot.transformers.trestle_rule import TrestleRule -class RulesTransformer(ABC): - """Abstract interface for transformers for rules""" +class ToRulesTransformer(TransformerBase): + """Abstract interface for transforming to rule data.""" @abstractmethod - def transform_to_rule(self, data: Any) -> TrestleRule: + def transform(self, data: Any) -> TrestleRule: """Transform to rule data.""" + +class FromRulesTransformer(TransformerBase): + """Abstract interface for transforming from rule data.""" + @abstractmethod - def transform_from_rule(self, rule: TrestleRule) -> Any: + def transform(self, rule: TrestleRule) -> Any: """Transform from rule data.""" diff --git a/trestlebot/transformers/csv_to_yaml.py b/trestlebot/transformers/csv_to_yaml.py index 7c005e32..f1d08172 100644 --- a/trestlebot/transformers/csv_to_yaml.py +++ b/trestlebot/transformers/csv_to_yaml.py @@ -19,16 +19,14 @@ import pathlib from typing import List -import ruamel.yaml as yaml - -from trestlebot.transformers.csv_transformer import RulesCSVTransformer +from trestlebot.transformers.csv_transformer import ToRulesCSVTransformer from trestlebot.transformers.trestle_rule import ( ComponentInfo, Control, Profile, TrestleRule, ) -from trestlebot.transformers.yaml_transformer import RulesYAMLTransformer +from trestlebot.transformers.yaml_transformer import FromRulesYAMLTransformer class YAMLBuilder: @@ -37,8 +35,8 @@ class YAMLBuilder: def __init__(self) -> None: """Initialize.""" self._rules: List[TrestleRule] = [] - self._yaml_transformer = RulesYAMLTransformer() - self._csv_transformer = RulesCSVTransformer() + self._yaml_transformer = FromRulesYAMLTransformer() + self._csv_transformer = ToRulesCSVTransformer() def read_from_csv(self, filepath: pathlib.Path) -> None: """Read from a CSV file and populate self._rules.""" @@ -46,21 +44,21 @@ def read_from_csv(self, filepath: pathlib.Path) -> None: with open(filepath, mode="r", newline="") as csv_file: reader = csv.DictReader(csv_file) for row in reader: - self._rules.append(self._csv_transformer.transform_to_rule(row)) - except Exception as e: - raise CSVReadError(f"Failed to read from CSV file: {e}") + self._rules.append(self._csv_transformer.transform(row)) + except FileNotFoundError: + raise CSVReadError(f"File not found: {filepath}") + except csv.Error as e: + raise CSVReadError(f"CSV reading error: {e}") def write_to_yaml(self, filepath: pathlib.Path) -> None: """Write the rules to a YAML file.""" try: - with open(filepath, "w") as yaml_file: - yaml.dump( - [ - self._yaml_transformer.transform_from_rule(rule) - for rule in self._rules - ], - yaml_file, - ) + with open(filepath, mode="w") as yaml_file: + yaml_file.write("---\n") + for rule in self._rules: + yaml_str = self._yaml_transformer.transform(rule).getvalue() + yaml_file.write(yaml_str) + yaml_file.write("\n") except Exception as e: raise YAMLWriteError(f"Failed to write rules to YAML file: {e}") @@ -81,9 +79,7 @@ def write_default_trestle_rule_keys(self, filepath: pathlib.Path) -> None: include_controls=[Control(id="example")], ), ) - yaml_blob = self._yaml_transformer.transform_from_rule(test_rule) - with open(filepath, "w") as yaml_file: - yaml_file.write(yaml_blob) + self._yaml_transformer.write_to_file(test_rule, filepath) except Exception as e: raise YAMLWriteError( f"Failed to write empty TrestleRule keys to YAML file: {e}" diff --git a/trestlebot/transformers/csv_transformer.py b/trestlebot/transformers/csv_transformer.py index 5ff8a526..f3bae64f 100644 --- a/trestlebot/transformers/csv_transformer.py +++ b/trestlebot/transformers/csv_transformer.py @@ -42,7 +42,10 @@ ) from trestlebot import const -from trestlebot.transformers.base_transformer import RulesTransformer +from trestlebot.transformers.base_transformer import ( + FromRulesTransformer, + ToRulesTransformer, +) from trestlebot.transformers.trestle_rule import ( ComponentInfo, Control, @@ -55,7 +58,7 @@ logger = logging.getLogger(__name__) -class RulesCSVTransformer(RulesTransformer): +class ToRulesCSVTransformer(ToRulesTransformer): """ Interface for CSV transformer to Rules model. @@ -67,7 +70,7 @@ def __init__(self) -> None: """Initialize.""" super().__init__() - def transform_to_rule(self, row: Dict[str, str]) -> TrestleRule: + def transform(self, row: Dict[str, str]) -> TrestleRule: """Transform a CSV row to a TrestleRule object.""" rule_info = self._extract_rule_info(row) profile = self._extract_profile(row) @@ -82,22 +85,6 @@ def transform_to_rule(self, row: Dict[str, str]) -> TrestleRule: profile=profile, ) - def transform_from_rule(self, rule: TrestleRule) -> Dict[str, str]: - """Transforms TrestleRule into a row of CSV.""" - rule_dict: Dict[str, str] = { - RULE_ID: rule.name, - RULE_DESCRIPTION: rule.description, - NAMESPACE: TRESTLE_GENERIC_NS, - } - merged_dict = { - **rule_dict, - **self._add_profile(rule.profile), - **self._add_component_info(rule.component), - } - if rule.parameter is not None: - merged_dict.update(self._add_parameter(rule.parameter)) - return merged_dict - def _extract_rule_info(self, row: Dict[str, str]) -> Dict[str, str]: """Extract rule information from a CSV row.""" return { @@ -138,6 +125,35 @@ def _extract_component_info(self, row: Dict[str, str]) -> ComponentInfo: description=row.get(csv_to_oscal_cd.COMPONENT_DESCRIPTION, ""), ) + +class FromRulesCSVTransformer(FromRulesTransformer): + """ + Interface for CSV transformer from Rules model. + + Notes: This will transform individual rows of CSV to and from a + Trestle object with row compliance with the Trestle CSV requirements. + """ + + def __init__(self) -> None: + """Initialize.""" + super().__init__() + + def transform(self, rule: TrestleRule) -> Dict[str, str]: + """Transforms TrestleRule into a row of CSV.""" + rule_dict: Dict[str, str] = { + RULE_ID: rule.name, + RULE_DESCRIPTION: rule.description, + NAMESPACE: TRESTLE_GENERIC_NS, + } + merged_dict = { + **rule_dict, + **self._add_profile(rule.profile), + **self._add_component_info(rule.component), + } + if rule.parameter is not None: + merged_dict.update(self._add_parameter(rule.parameter)) + return merged_dict + def _add_profile(self, profile: Profile) -> Dict[str, str]: """Add a profile to the CSV Row.""" controls_list: List[str] = [control.id for control in profile.include_controls] @@ -174,7 +190,7 @@ class CSVBuilder: def __init__(self) -> None: """Initialize.""" self._csv_columns: CsvColumn = CsvColumn() - self._transformer: RulesCSVTransformer = RulesCSVTransformer() + self._transformer: FromRulesTransformer = FromRulesCSVTransformer() self._rows: List[Dict[str, str]] = [] @property @@ -184,7 +200,7 @@ def row_count(self) -> int: def add_row(self, rule: TrestleRule) -> None: """Add a row to the CSV.""" - row = self._transformer.transform_from_rule(rule) + row = self._transformer.transform(rule) self.validate_row(row) self._rows.append(row) diff --git a/trestlebot/transformers/yaml_transformer.py b/trestlebot/transformers/yaml_transformer.py index d967e33b..f03debf0 100644 --- a/trestlebot/transformers/yaml_transformer.py +++ b/trestlebot/transformers/yaml_transformer.py @@ -16,6 +16,7 @@ """YAML transformer for rule authoring.""" import logging +import pathlib from io import StringIO from typing import Any, Dict @@ -24,8 +25,9 @@ from trestlebot import const from trestlebot.transformers.base_transformer import ( - RulesTransformer, + FromRulesTransformer, RulesTransformerException, + ToRulesTransformer, ) from trestlebot.transformers.trestle_rule import ( ComponentInfo, @@ -38,14 +40,14 @@ logger = logging.getLogger(__name__) -class RulesYAMLTransformer(RulesTransformer): +class ToRulesYAMLTransformer(ToRulesTransformer): """Interface for YAML transformer to Rules model.""" def __init__(self) -> None: """Initialize.""" super().__init__() - def transform_to_rule(self, blob: str) -> TrestleRule: + def transform(self, blob: str) -> TrestleRule: """Transform YAML data into a TrestleRule object.""" try: yaml = YAML(typ="safe") @@ -78,16 +80,44 @@ def transform_to_rule(self, blob: str) -> TrestleRule: raise RulesTransformerException(f"Missing key in YAML file: {e}") except ValidationError as e: raise RulesTransformerException(f"Invalid YAML file: {e}") - except Exception as e: - raise RuntimeError(e) return rule_info_instance - def transform_from_rule(self, rule: TrestleRule) -> str: - """Transform TrestleRule object into YAML data.""" + +class FromRulesYAMLTransformer(FromRulesTransformer): + """ + Interface for YAML transformer from Rules model. + """ + + def transform(self, rule: TrestleRule) -> StringIO: + """ + Transform TrestleRule object into YAML data. + + Notes: + Currently, this method is for simply transforming + TrestleRule objects in memory into YAML data. The data structure + is small so this should have minimal performance impact. This is done to + comply with the contract of the RulesTransformer interface. + """ + + rule_info: Dict[str, Any] = self._to_rule_info(rule) + yaml_obj = YAML() + yaml_obj.default_flow_style = False + yaml_stream = StringIO() + yaml_obj.dump(rule_info, yaml_stream) + + return yaml_stream + + def write_to_file(self, rule: TrestleRule, file_path: pathlib.Path) -> None: + """Write TrestleRule object to YAML file.""" + rule_info: Dict[str, Any] = self._to_rule_info(rule) yaml_obj = YAML() yaml_obj.default_flow_style = False + yaml_obj.dump(rule_info, file_path) + @staticmethod + def _to_rule_info(rule: TrestleRule) -> Dict[str, Any]: + """Convert YAML data to rule info.""" rule_info: Dict[str, Any] = { const.RULE_INFO_TAG: { const.NAME: rule.name, @@ -103,8 +133,4 @@ def transform_from_rule(self, rule: TrestleRule) -> str: rule_info[const.RULE_INFO_TAG][const.PARAMETER] = rule.parameter.dict( by_alias=True, exclude_unset=True ) - - yaml_stream = StringIO() - yaml_obj.dump(rule_info, yaml_stream) - - return yaml_stream.getvalue() + return rule_info