From 828277d5f7429e4e422af210fece711fa83db75b Mon Sep 17 00:00:00 2001 From: MiaAltieri Date: Tue, 26 Mar 2024 16:18:13 +0000 Subject: [PATCH 1/2] test rotating certs --- .../external_relations/test_charm.py | 4 +- tests/integration/test_charm.py | 4 +- tests/integration/tls_tests/helpers.py | 167 ++++++++++++++++++ tests/integration/tls_tests/test_tls.py | 85 ++++++++- 4 files changed, 247 insertions(+), 13 deletions(-) diff --git a/tests/integration/external_relations/test_charm.py b/tests/integration/external_relations/test_charm.py index 29ad6834..a1971b29 100644 --- a/tests/integration/external_relations/test_charm.py +++ b/tests/integration/external_relations/test_charm.py @@ -36,14 +36,14 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: MONGODB_CHARM_NAME, application_name=CONFIG_SERVER_APP_NAME, channel="6/edge", - revision=142, + revision=164, config={"role": "config-server"}, ) await ops_test.model.deploy( MONGODB_CHARM_NAME, application_name=SHARD_APP_NAME, channel="6/edge", - revision=142, + revision=164, config={"role": "shard"}, ) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 3fbbe0f0..ed41cf23 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -45,14 +45,14 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: MONGODB_CHARM_NAME, application_name=CONFIG_SERVER_APP_NAME, channel="6/edge", - revision=142, + revision=164, config={"role": "config-server"}, ) await ops_test.model.deploy( MONGODB_CHARM_NAME, application_name=SHARD_APP_NAME, channel="6/edge", - revision=142, + revision=164, config={"role": "shard"}, ) diff --git a/tests/integration/tls_tests/helpers.py b/tests/integration/tls_tests/helpers.py index c9531f9f..c52fc979 100644 --- a/tests/integration/tls_tests/helpers.py +++ b/tests/integration/tls_tests/helpers.py @@ -5,18 +5,28 @@ from pytest_operator.plugin import OpsTest from ..helpers import get_application_relation_data, get_secret_data from tenacity import RetryError, Retrying, stop_after_attempt, wait_exponential +from datetime import datetime +from typing import Optional, Dict +import json +import ops + MONGODB_SNAP_DATA_DIR = "/var/snap/charmed-mongodb/current" MONGOD_CONF_DIR = f"{MONGODB_SNAP_DATA_DIR}/etc/mongod" MONGO_COMMON_DIR = "/var/snap/charmed-mongodb/common" EXTERNAL_PEM_PATH = f"{MONGOD_CONF_DIR}/external-cert.pem" EXTERNAL_CERT_PATH = f"{MONGOD_CONF_DIR}/external-ca.crt" +INTERNAL_CERT_PATH = f"{MONGOD_CONF_DIR}/internal-ca.crt" MONGO_SHELL = "charmed-mongodb.mongosh" MONGOS_APP_NAME = "mongos" CERT_REL_NAME = "certificates" CERTS_APP_NAME = "self-signed-certificates" +class ProcessError(Exception): + """Raised when a process fails.""" + + async def check_mongos_tls_disabled(ops_test: OpsTest) -> None: # check mongos is running with TLS enabled for unit in ops_test.model.applications[MONGOS_APP_NAME].units: @@ -81,3 +91,160 @@ async def check_tls(ops_test, unit, enabled) -> None: return True except RetryError: return False + + +async def time_file_created(ops_test: OpsTest, unit_name: str, path: str) -> int: + """Returns the unix timestamp of when a file was created on a specified unit.""" + time_cmd = f"exec --unit {unit_name} -- ls -l --time-style=full-iso {path} " + return_code, ls_output, _ = await ops_test.juju(*time_cmd.split()) + + if return_code != 0: + raise ProcessError( + "Expected time command %s to succeed instead it failed: %s", + time_cmd, + return_code, + ) + + return process_ls_time(ls_output) + + +async def time_process_started( + ops_test: OpsTest, unit_name: str, process_name: str +) -> int: + """Retrieves the time that a given process started according to systemd.""" + time_cmd = f"exec --unit {unit_name} -- systemctl show {process_name} --property=ActiveEnterTimestamp" + return_code, systemctl_output, _ = await ops_test.juju(*time_cmd.split()) + + if return_code != 0: + raise ProcessError( + "Expected time command %s to succeed instead it failed: %s", + time_cmd, + return_code, + ) + + return process_systemctl_time(systemctl_output) + + +async def check_certs_correctly_distributed( + ops_test: OpsTest, unit: ops.Unit, app_name=None +) -> None: + """Comparing expected vs distributed certificates. + + Verifying certificates downloaded on the charm against the ones distributed by the TLS operator + """ + app_name = app_name + app_secret_id = await get_secret_id(ops_test, app_name) + unit_secret_id = await get_secret_id(ops_test, unit.name) + app_secret_content = await get_secret_content(ops_test, app_secret_id) + unit_secret_content = await get_secret_content(ops_test, unit_secret_id) + app_current_crt = app_secret_content["csr-secret"] + unit_current_crt = unit_secret_content["csr-secret"] + + # Get the values for certs from the relation, as provided by TLS Charm + certificates_raw_data = await get_application_relation_data( + ops_test, app_name, CERT_REL_NAME, "certificates" + ) + certificates_data = json.loads(certificates_raw_data) + + external_item = [ + data + for data in certificates_data + if data["certificate_signing_request"].rstrip() == unit_current_crt.rstrip() + ][0] + internal_item = [ + data + for data in certificates_data + if data["certificate_signing_request"].rstrip() == app_current_crt.rstrip() + ][0] + + # Get a local copy of the external cert + external_copy_path = await scp_file_preserve_ctime( + ops_test, unit.name, EXTERNAL_CERT_PATH + ) + + # Get the external cert value from the relation + relation_external_cert = "\n".join(external_item["chain"]) + + # CHECK: Compare if they are the same + with open(external_copy_path) as f: + external_contents_file = f.read() + assert relation_external_cert == external_contents_file + + # Get a local copy of the internal cert + internal_copy_path = await scp_file_preserve_ctime( + ops_test, unit.name, INTERNAL_CERT_PATH + ) + + # Get the external cert value from the relation + relation_internal_cert = "\n".join(internal_item["chain"]) + + # CHECK: Compare if they are the same + with open(internal_copy_path) as f: + internal_contents_file = f.read() + assert relation_internal_cert == internal_contents_file + + +async def scp_file_preserve_ctime(ops_test: OpsTest, unit_name: str, path: str) -> int: + """Returns the unix timestamp of when a file was created on a specified unit.""" + # Retrieving the file + filename = path.split("/")[-1] + complete_command = f"scp --container mongod {unit_name}:{path} {filename}" + return_code, _, stderr = await ops_test.juju(*complete_command.split()) + + if return_code != 0: + raise ProcessError( + "Expected command %s to succeed instead it failed: %s; %s", + complete_command, + return_code, + stderr, + ) + + return f"{filename}" + + +def process_ls_time(ls_output): + """Parse time representation as returned by the 'ls' command.""" + time_as_str = "T".join(ls_output.split("\n")[0].split(" ")[5:7]) + # further strip down additional milliseconds + time_as_str = time_as_str[0:-3] + d = datetime.strptime(time_as_str, "%Y-%m-%dT%H:%M:%S.%f") + return d + + +def process_systemctl_time(systemctl_output): + """Parse time representation as returned by the 'systemctl' command.""" + "ActiveEnterTimestamp=Thu 2022-09-22 10:00:00 UTC" + time_as_str = "T".join(systemctl_output.split("=")[1].split(" ")[1:3]) + d = datetime.strptime(time_as_str, "%Y-%m-%dT%H:%M:%S") + return d + + +async def get_secret_id(ops_test, app_or_unit: Optional[str] = None) -> str: + """Retrieve secert ID for an app or unit.""" + complete_command = "list-secrets" + + prefix = "" + if app_or_unit: + if app_or_unit[-1].isdigit(): + # it's a unit + app_or_unit = "-".join(app_or_unit.split("/")) + prefix = "unit-" + else: + prefix = "application-" + complete_command += f" --owner {prefix}{app_or_unit}" + + _, stdout, _ = await ops_test.juju(*complete_command.split()) + output_lines_split = [line.split() for line in stdout.split("\n")] + if app_or_unit: + return [line[0] for line in output_lines_split if app_or_unit in line][0] + else: + return output_lines_split[1][0] + + +async def get_secret_content(ops_test, secret_id) -> Dict[str, str]: + """Retrieve contents of a Juju Secret.""" + secret_id = secret_id.split("/")[-1] + complete_command = f"show-secret {secret_id} --reveal --format=json" + _, stdout, _ = await ops_test.juju(*complete_command.split()) + data = json.loads(stdout) + return data[secret_id]["content"]["Data"] diff --git a/tests/integration/tls_tests/test_tls.py b/tests/integration/tls_tests/test_tls.py index 0510cb1a..7ca82189 100644 --- a/tests/integration/tls_tests/test_tls.py +++ b/tests/integration/tls_tests/test_tls.py @@ -7,9 +7,14 @@ check_mongos_tls_enabled, check_mongos_tls_disabled, toggle_tls_mongos, + EXTERNAL_CERT_PATH, + INTERNAL_CERT_PATH, + check_certs_correctly_distributed, + time_file_created, + time_process_started, ) - +MONGOS_SERVICE = "snap.charmed-mongodb.mongos.service" APPLICATION_APP_NAME = "application" MONGOS_APP_NAME = "mongos" MONGODB_CHARM_NAME = "mongodb" @@ -25,7 +30,6 @@ TIMEOUT = 15 * 60 -@pytest.mark.skip("Wait new MongoDB charm is published.") @pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -35,12 +39,11 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await deploy_tls(ops_test) -@pytest.mark.skip("Wait new MongoDB charm is published.") @pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_mongos_tls_enabled(ops_test: OpsTest) -> None: """Tests that mongos charm can enable TLS.""" - # await integrate_mongos_with_tls(ops_test) + await integrate_mongos_with_tls(ops_test) await ops_test.model.wait_for_idle( apps=[MONGOS_APP_NAME], @@ -61,7 +64,12 @@ async def test_mongos_tls_enabled(ops_test: OpsTest) -> None: await check_mongos_tls_enabled(ops_test) -@pytest.mark.skip("Wait new MongoDB charm is published.") +@pytest.mark.group(1) +@pytest.mark.abort_on_fail +async def test_mongos_rotate_certs(ops_test: OpsTest) -> None: + await rotate_and_verify_certs(ops_test) + + @pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_mongos_tls_disabled(ops_test: OpsTest) -> None: @@ -75,7 +83,6 @@ async def test_mongos_tls_disabled(ops_test: OpsTest) -> None: ), "mongos fails to report TLS inconsistencies." -@pytest.mark.skip("Wait new MongoDB charm is published.") @pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_tls_reenabled(ops_test: OpsTest) -> None: @@ -84,7 +91,6 @@ async def test_tls_reenabled(ops_test: OpsTest) -> None: await check_mongos_tls_enabled(ops_test) -@pytest.mark.skip("Wait new MongoDB charm is published.") @pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_mongos_tls_ca_mismatch(ops_test: OpsTest) -> None: @@ -138,14 +144,14 @@ async def deploy_cluster(ops_test: OpsTest) -> None: MONGODB_CHARM_NAME, application_name=CONFIG_SERVER_APP_NAME, channel="6/edge", - revision=142, + revision=164, config={"role": "config-server"}, ) await ops_test.model.deploy( MONGODB_CHARM_NAME, application_name=SHARD_APP_NAME, channel="6/edge", - revision=142, + revision=164, config={"role": "shard"}, ) @@ -234,3 +240,64 @@ async def integrate_cluster_with_tls(ops_test: OpsTest) -> None: raise_on_blocked=False, status="active", ) + + +async def rotate_and_verify_certs(ops_test: OpsTest) -> None: + """Verifies that each unit is able to rotate their TLS certificates.""" + original_tls_times = {} + for unit in ops_test.model.applications[MONGOS_APP_NAME].units: + original_tls_times[unit.name] = {} + original_tls_times[unit.name]["external_cert"] = await time_file_created( + ops_test, unit.name, EXTERNAL_CERT_PATH + ) + original_tls_times[unit.name]["internal_cert"] = await time_file_created( + ops_test, unit.name, INTERNAL_CERT_PATH + ) + original_tls_times[unit.name]["mongos_service"] = await time_process_started( + ops_test, unit.name, MONGOS_SERVICE + ) + check_certs_correctly_distributed(ops_test, unit) + + # set external and internal key using auto-generated key for each unit + for unit in ops_test.model.applications[MONGOS_APP_NAME].units: + action = await unit.run_action(action_name="set-tls-private-key") + action = await action.wait() + assert action.status == "completed", "setting external and internal key failed." + + # wait for certificate to be available and processed. Can get receive two certificate + # available events and restart twice so we want to ensure we are idle for at least 1 minute + await ops_test.model.wait_for_idle( + apps=[MONGOS_APP_NAME], status="active", timeout=1000, idle_period=60 + ) + + # After updating both the external key and the internal key a new certificate request will be + # made; then the certificates should be available and updated. + for unit in ops_test.model.applications[MONGOS_APP_NAME].units: + new_external_cert_time = await time_file_created( + ops_test, unit.name, EXTERNAL_CERT_PATH + ) + new_internal_cert_time = await time_file_created( + ops_test, unit.name, INTERNAL_CERT_PATH + ) + new_mongos_service_time = await time_process_started( + ops_test, unit.name, MONGOS_SERVICE + ) + + check_certs_correctly_distributed(ops_test, unit, app_name=MONGOS_APP_NAME) + + assert ( + new_external_cert_time > original_tls_times[unit.name]["external_cert"] + ), f"external cert for {unit.name} was not updated." + assert ( + new_internal_cert_time > original_tls_times[unit.name]["internal_cert"] + ), f"internal cert for {unit.name} was not updated." + + # Once the certificate requests are processed and updated the mongos.service should be + # restarted + assert ( + new_mongos_service_time > original_tls_times[unit.name]["mongos_service"] + ), f"mongos service for {unit.name} was not restarted." + + # Verify that TLS is functioning on all units. + for unit in ops_test.model.applications[MONGOS_APP_NAME].units: + check_mongos_tls_enabled(ops_test) From 5b132e912ca4878de5c762b993f5c55a9e72d568 Mon Sep 17 00:00:00 2001 From: MiaAltieri Date: Tue, 9 Apr 2024 07:42:13 +0000 Subject: [PATCH 2/2] pr feedback --- tests/integration/tls_tests/helpers.py | 7 +++++ tests/integration/tls_tests/test_tls.py | 34 +++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/tests/integration/tls_tests/helpers.py b/tests/integration/tls_tests/helpers.py index c52fc979..2376b113 100644 --- a/tests/integration/tls_tests/helpers.py +++ b/tests/integration/tls_tests/helpers.py @@ -248,3 +248,10 @@ async def get_secret_content(ops_test, secret_id) -> Dict[str, str]: _, stdout, _ = await ops_test.juju(*complete_command.split()) data = json.loads(stdout) return data[secret_id]["content"]["Data"] + + +async def get_file_contents(ops_test: OpsTest, unit: str, filepath: str) -> str: + """Returns the contents of the provided filepath.""" + mv_cmd = f"exec --unit {unit.name} sudo cat {filepath} " + _, stdout, _ = await ops_test.juju(*mv_cmd.split()) + return stdout diff --git a/tests/integration/tls_tests/test_tls.py b/tests/integration/tls_tests/test_tls.py index 7ca82189..6636f8fa 100644 --- a/tests/integration/tls_tests/test_tls.py +++ b/tests/integration/tls_tests/test_tls.py @@ -9,6 +9,7 @@ toggle_tls_mongos, EXTERNAL_CERT_PATH, INTERNAL_CERT_PATH, + get_file_contents, check_certs_correctly_distributed, time_file_created, time_process_started, @@ -244,16 +245,23 @@ async def integrate_cluster_with_tls(ops_test: OpsTest) -> None: async def rotate_and_verify_certs(ops_test: OpsTest) -> None: """Verifies that each unit is able to rotate their TLS certificates.""" - original_tls_times = {} + original_tls_info = {} for unit in ops_test.model.applications[MONGOS_APP_NAME].units: - original_tls_times[unit.name] = {} - original_tls_times[unit.name]["external_cert"] = await time_file_created( + original_tls_info[unit.name] = {} + + original_tls_info[unit.name][ + "external_cert_contents" + ] = await get_file_contents(ops_test, unit, EXTERNAL_CERT_PATH) + original_tls_info[unit.name][ + "internal_cert_contents" + ] = await get_file_contents(ops_test, unit, INTERNAL_CERT_PATH) + original_tls_info[unit.name]["external_cert_time"] = await time_file_created( ops_test, unit.name, EXTERNAL_CERT_PATH ) - original_tls_times[unit.name]["internal_cert"] = await time_file_created( + original_tls_info[unit.name]["internal_cert_time"] = await time_file_created( ops_test, unit.name, INTERNAL_CERT_PATH ) - original_tls_times[unit.name]["mongos_service"] = await time_process_started( + original_tls_info[unit.name]["mongos_service"] = await time_process_started( ops_test, unit.name, MONGOS_SERVICE ) check_certs_correctly_distributed(ops_test, unit) @@ -273,6 +281,9 @@ async def rotate_and_verify_certs(ops_test: OpsTest) -> None: # After updating both the external key and the internal key a new certificate request will be # made; then the certificates should be available and updated. for unit in ops_test.model.applications[MONGOS_APP_NAME].units: + new_external_cert = await get_file_contents(ops_test, unit, EXTERNAL_CERT_PATH) + new_internal_cert = await get_file_contents(ops_test, unit, INTERNAL_CERT_PATH) + new_external_cert_time = await time_file_created( ops_test, unit.name, EXTERNAL_CERT_PATH ) @@ -286,16 +297,23 @@ async def rotate_and_verify_certs(ops_test: OpsTest) -> None: check_certs_correctly_distributed(ops_test, unit, app_name=MONGOS_APP_NAME) assert ( - new_external_cert_time > original_tls_times[unit.name]["external_cert"] + new_external_cert != original_tls_info[unit.name]["external_cert_contents"] + ), "external cert not rotated" + + assert ( + new_internal_cert != original_tls_info[unit.name]["external_cert_contents"] + ), "external cert not rotated" + assert ( + new_external_cert_time > original_tls_info[unit.name]["external_cert_time"] ), f"external cert for {unit.name} was not updated." assert ( - new_internal_cert_time > original_tls_times[unit.name]["internal_cert"] + new_internal_cert_time > original_tls_info[unit.name]["internal_cert_time"] ), f"internal cert for {unit.name} was not updated." # Once the certificate requests are processed and updated the mongos.service should be # restarted assert ( - new_mongos_service_time > original_tls_times[unit.name]["mongos_service"] + new_mongos_service_time > original_tls_info[unit.name]["mongos_service"] ), f"mongos service for {unit.name} was not restarted." # Verify that TLS is functioning on all units.