Merge pull request #28 from canonical/add-rotate-cert-test

test rotating certs

tests/integration/external_relations/test_charm.py

@@ -36,14 +36,14 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
         MONGODB_CHARM_NAME,
         application_name=CONFIG_SERVER_APP_NAME,
         channel="6/edge",
-        revision=142,
+        revision=164,
         config={"role": "config-server"},
     )
     await ops_test.model.deploy(
         MONGODB_CHARM_NAME,
         application_name=SHARD_APP_NAME,
         channel="6/edge",
-        revision=142,
+        revision=164,
         config={"role": "shard"},
     )
 

tests/integration/test_charm.py

@@ -45,14 +45,14 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
         MONGODB_CHARM_NAME,
         application_name=CONFIG_SERVER_APP_NAME,
         channel="6/edge",
-        revision=142,
+        revision=164,
         config={"role": "config-server"},
     )
     await ops_test.model.deploy(
         MONGODB_CHARM_NAME,
         application_name=SHARD_APP_NAME,
         channel="6/edge",
-        revision=142,
+        revision=164,
         config={"role": "shard"},
     )
 

tests/integration/tls_tests/helpers.py

@@ -5,18 +5,28 @@
 from pytest_operator.plugin import OpsTest
 from ..helpers import get_application_relation_data, get_secret_data
 from tenacity import RetryError, Retrying, stop_after_attempt, wait_exponential
+from datetime import datetime
+from typing import Optional, Dict
+import json
+import ops
+
 
 MONGODB_SNAP_DATA_DIR = "/var/snap/charmed-mongodb/current"
 MONGOD_CONF_DIR = f"{MONGODB_SNAP_DATA_DIR}/etc/mongod"
 MONGO_COMMON_DIR = "/var/snap/charmed-mongodb/common"
 EXTERNAL_PEM_PATH = f"{MONGOD_CONF_DIR}/external-cert.pem"
 EXTERNAL_CERT_PATH = f"{MONGOD_CONF_DIR}/external-ca.crt"
+INTERNAL_CERT_PATH = f"{MONGOD_CONF_DIR}/internal-ca.crt"
 MONGO_SHELL = "charmed-mongodb.mongosh"
 MONGOS_APP_NAME = "mongos"
 CERT_REL_NAME = "certificates"
 CERTS_APP_NAME = "self-signed-certificates"
 
 
+class ProcessError(Exception):
+    """Raised when a process fails."""
+
+
 async def check_mongos_tls_disabled(ops_test: OpsTest) -> None:
     # check mongos is running with TLS enabled
     for unit in ops_test.model.applications[MONGOS_APP_NAME].units:
@@ -81,3 +91,167 @@ async def check_tls(ops_test, unit, enabled) -> None:
                 return True
     except RetryError:
         return False
+
+
+async def time_file_created(ops_test: OpsTest, unit_name: str, path: str) -> int:
+    """Returns the unix timestamp of when a file was created on a specified unit."""
+    time_cmd = f"exec --unit {unit_name} --  ls -l --time-style=full-iso {path} "
+    return_code, ls_output, _ = await ops_test.juju(*time_cmd.split())
+
+    if return_code != 0:
+        raise ProcessError(
+            "Expected time command %s to succeed instead it failed: %s",
+            time_cmd,
+            return_code,
+        )
+
+    return process_ls_time(ls_output)
+
+
+async def time_process_started(
+    ops_test: OpsTest, unit_name: str, process_name: str
+) -> int:
+    """Retrieves the time that a given process started according to systemd."""
+    time_cmd = f"exec --unit {unit_name} --  systemctl show {process_name} --property=ActiveEnterTimestamp"
+    return_code, systemctl_output, _ = await ops_test.juju(*time_cmd.split())
+
+    if return_code != 0:
+        raise ProcessError(
+            "Expected time command %s to succeed instead it failed: %s",
+            time_cmd,
+            return_code,
+        )
+
+    return process_systemctl_time(systemctl_output)
+
+
+async def check_certs_correctly_distributed(
+    ops_test: OpsTest, unit: ops.Unit, app_name=None
+) -> None:
+    """Comparing expected vs distributed certificates.
+
+    Verifying certificates downloaded on the charm against the ones distributed by the TLS operator
+    """
+    app_name = app_name
+    app_secret_id = await get_secret_id(ops_test, app_name)
+    unit_secret_id = await get_secret_id(ops_test, unit.name)
+    app_secret_content = await get_secret_content(ops_test, app_secret_id)
+    unit_secret_content = await get_secret_content(ops_test, unit_secret_id)
+    app_current_crt = app_secret_content["csr-secret"]
+    unit_current_crt = unit_secret_content["csr-secret"]
+
+    # Get the values for certs from the relation, as provided by TLS Charm
+    certificates_raw_data = await get_application_relation_data(
+        ops_test, app_name, CERT_REL_NAME, "certificates"
+    )
+    certificates_data = json.loads(certificates_raw_data)
+
+    external_item = [
+        data
+        for data in certificates_data
+        if data["certificate_signing_request"].rstrip() == unit_current_crt.rstrip()
+    ][0]
+    internal_item = [
+        data
+        for data in certificates_data
+        if data["certificate_signing_request"].rstrip() == app_current_crt.rstrip()
+    ][0]
+
+    # Get a local copy of the external cert
+    external_copy_path = await scp_file_preserve_ctime(
+        ops_test, unit.name, EXTERNAL_CERT_PATH
+    )
+
+    # Get the external cert value from the relation
+    relation_external_cert = "\n".join(external_item["chain"])
+
+    # CHECK: Compare if they are the same
+    with open(external_copy_path) as f:
+        external_contents_file = f.read()
+        assert relation_external_cert == external_contents_file
+
+    # Get a local copy of the internal cert
+    internal_copy_path = await scp_file_preserve_ctime(
+        ops_test, unit.name, INTERNAL_CERT_PATH
+    )
+
+    # Get the external cert value from the relation
+    relation_internal_cert = "\n".join(internal_item["chain"])
+
+    # CHECK: Compare if they are the same
+    with open(internal_copy_path) as f:
+        internal_contents_file = f.read()
+        assert relation_internal_cert == internal_contents_file
+
+
+async def scp_file_preserve_ctime(ops_test: OpsTest, unit_name: str, path: str) -> int:
+    """Returns the unix timestamp of when a file was created on a specified unit."""
+    # Retrieving the file
+    filename = path.split("/")[-1]
+    complete_command = f"scp --container mongod {unit_name}:{path} {filename}"
+    return_code, _, stderr = await ops_test.juju(*complete_command.split())
+
+    if return_code != 0:
+        raise ProcessError(
+            "Expected command %s to succeed instead it failed: %s; %s",
+            complete_command,
+            return_code,
+            stderr,
+        )
+
+    return f"{filename}"
+
+
+def process_ls_time(ls_output):
+    """Parse time representation as returned by the 'ls' command."""
+    time_as_str = "T".join(ls_output.split("\n")[0].split(" ")[5:7])
+    # further strip down additional milliseconds
+    time_as_str = time_as_str[0:-3]
+    d = datetime.strptime(time_as_str, "%Y-%m-%dT%H:%M:%S.%f")
+    return d
+
+
+def process_systemctl_time(systemctl_output):
+    """Parse time representation as returned by the 'systemctl' command."""
+    "ActiveEnterTimestamp=Thu 2022-09-22 10:00:00 UTC"
+    time_as_str = "T".join(systemctl_output.split("=")[1].split(" ")[1:3])
+    d = datetime.strptime(time_as_str, "%Y-%m-%dT%H:%M:%S")
+    return d
+
+
+async def get_secret_id(ops_test, app_or_unit: Optional[str] = None) -> str:
+    """Retrieve secert ID for an app or unit."""
+    complete_command = "list-secrets"
+
+    prefix = ""
+    if app_or_unit:
+        if app_or_unit[-1].isdigit():
+            # it's a unit
+            app_or_unit = "-".join(app_or_unit.split("/"))
+            prefix = "unit-"
+        else:
+            prefix = "application-"
+        complete_command += f" --owner {prefix}{app_or_unit}"
+
+    _, stdout, _ = await ops_test.juju(*complete_command.split())
+    output_lines_split = [line.split() for line in stdout.split("\n")]
+    if app_or_unit:
+        return [line[0] for line in output_lines_split if app_or_unit in line][0]
+    else:
+        return output_lines_split[1][0]
+
+
+async def get_secret_content(ops_test, secret_id) -> Dict[str, str]:
+    """Retrieve contents of a Juju Secret."""
+    secret_id = secret_id.split("/")[-1]
+    complete_command = f"show-secret {secret_id} --reveal --format=json"
+    _, stdout, _ = await ops_test.juju(*complete_command.split())
+    data = json.loads(stdout)
+    return data[secret_id]["content"]["Data"]
+
+
+async def get_file_contents(ops_test: OpsTest, unit: str, filepath: str) -> str:
+    """Returns the contents of the provided filepath."""
+    mv_cmd = f"exec --unit {unit.name} sudo cat {filepath} "
+    _, stdout, _ = await ops_test.juju(*mv_cmd.split())
+    return stdout

tests/integration/tls_tests/test_tls.py

@@ -7,9 +7,15 @@
     check_mongos_tls_enabled,
     check_mongos_tls_disabled,
     toggle_tls_mongos,
+    EXTERNAL_CERT_PATH,
+    INTERNAL_CERT_PATH,
+    get_file_contents,
+    check_certs_correctly_distributed,
+    time_file_created,
+    time_process_started,
 )
 
-
+MONGOS_SERVICE = "snap.charmed-mongodb.mongos.service"
 APPLICATION_APP_NAME = "application"
 MONGOS_APP_NAME = "mongos"
 MONGODB_CHARM_NAME = "mongodb"
@@ -25,7 +31,6 @@
 TIMEOUT = 15 * 60
 
 
-@pytest.mark.skip("Wait new MongoDB charm is published.")
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_build_and_deploy(ops_test: OpsTest) -> None:
@@ -35,12 +40,11 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
     await deploy_tls(ops_test)
 
 
-@pytest.mark.skip("Wait new MongoDB charm is published.")
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_mongos_tls_enabled(ops_test: OpsTest) -> None:
     """Tests that mongos charm can enable TLS."""
-    # await integrate_mongos_with_tls(ops_test)
+    await integrate_mongos_with_tls(ops_test)
 
     await ops_test.model.wait_for_idle(
         apps=[MONGOS_APP_NAME],
@@ -61,7 +65,12 @@ async def test_mongos_tls_enabled(ops_test: OpsTest) -> None:
     await check_mongos_tls_enabled(ops_test)
 
 
-@pytest.mark.skip("Wait new MongoDB charm is published.")
+@pytest.mark.group(1)
+@pytest.mark.abort_on_fail
+async def test_mongos_rotate_certs(ops_test: OpsTest) -> None:
+    await rotate_and_verify_certs(ops_test)
+
+
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_mongos_tls_disabled(ops_test: OpsTest) -> None:
@@ -75,7 +84,6 @@ async def test_mongos_tls_disabled(ops_test: OpsTest) -> None:
     ), "mongos fails to report TLS inconsistencies."
 
 
-@pytest.mark.skip("Wait new MongoDB charm is published.")
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_tls_reenabled(ops_test: OpsTest) -> None:
@@ -84,7 +92,6 @@ async def test_tls_reenabled(ops_test: OpsTest) -> None:
     await check_mongos_tls_enabled(ops_test)
 
 
-@pytest.mark.skip("Wait new MongoDB charm is published.")
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_mongos_tls_ca_mismatch(ops_test: OpsTest) -> None:
@@ -138,14 +145,14 @@ async def deploy_cluster(ops_test: OpsTest) -> None:
         MONGODB_CHARM_NAME,
         application_name=CONFIG_SERVER_APP_NAME,
         channel="6/edge",
-        revision=142,
+        revision=164,
         config={"role": "config-server"},
     )
     await ops_test.model.deploy(
         MONGODB_CHARM_NAME,
         application_name=SHARD_APP_NAME,
         channel="6/edge",
-        revision=142,
+        revision=164,
         config={"role": "shard"},
     )
 
@@ -234,3 +241,81 @@ async def integrate_cluster_with_tls(ops_test: OpsTest) -> None:
         raise_on_blocked=False,
         status="active",
     )
+
+
+async def rotate_and_verify_certs(ops_test: OpsTest) -> None:
+    """Verifies that each unit is able to rotate their TLS certificates."""
+    original_tls_info = {}
+    for unit in ops_test.model.applications[MONGOS_APP_NAME].units:
+        original_tls_info[unit.name] = {}
+
+        original_tls_info[unit.name][
+            "external_cert_contents"
+        ] = await get_file_contents(ops_test, unit, EXTERNAL_CERT_PATH)
+        original_tls_info[unit.name][
+            "internal_cert_contents"
+        ] = await get_file_contents(ops_test, unit, INTERNAL_CERT_PATH)
+        original_tls_info[unit.name]["external_cert_time"] = await time_file_created(
+            ops_test, unit.name, EXTERNAL_CERT_PATH
+        )
+        original_tls_info[unit.name]["internal_cert_time"] = await time_file_created(
+            ops_test, unit.name, INTERNAL_CERT_PATH
+        )
+        original_tls_info[unit.name]["mongos_service"] = await time_process_started(
+            ops_test, unit.name, MONGOS_SERVICE
+        )
+        check_certs_correctly_distributed(ops_test, unit)
+
+    # set external and internal key using auto-generated key for each unit
+    for unit in ops_test.model.applications[MONGOS_APP_NAME].units:
+        action = await unit.run_action(action_name="set-tls-private-key")
+        action = await action.wait()
+        assert action.status == "completed", "setting external and internal key failed."
+
+    # wait for certificate to be available and processed. Can get receive two certificate
+    # available events and restart twice so we want to ensure we are idle for at least 1 minute
+    await ops_test.model.wait_for_idle(
+        apps=[MONGOS_APP_NAME], status="active", timeout=1000, idle_period=60
+    )
+
+    # After updating both the external key and the internal key a new certificate request will be
+    # made; then the certificates should be available and updated.
+    for unit in ops_test.model.applications[MONGOS_APP_NAME].units:
+        new_external_cert = await get_file_contents(ops_test, unit, EXTERNAL_CERT_PATH)
+        new_internal_cert = await get_file_contents(ops_test, unit, INTERNAL_CERT_PATH)
+
+        new_external_cert_time = await time_file_created(
+            ops_test, unit.name, EXTERNAL_CERT_PATH
+        )
+        new_internal_cert_time = await time_file_created(
+            ops_test, unit.name, INTERNAL_CERT_PATH
+        )
+        new_mongos_service_time = await time_process_started(
+            ops_test, unit.name, MONGOS_SERVICE
+        )
+
+        check_certs_correctly_distributed(ops_test, unit, app_name=MONGOS_APP_NAME)
+
+        assert (
+            new_external_cert != original_tls_info[unit.name]["external_cert_contents"]
+        ), "external cert not rotated"
+
+        assert (
+            new_internal_cert != original_tls_info[unit.name]["external_cert_contents"]
+        ), "external cert not rotated"
+        assert (
+            new_external_cert_time > original_tls_info[unit.name]["external_cert_time"]
+        ), f"external cert for {unit.name} was not updated."
+        assert (
+            new_internal_cert_time > original_tls_info[unit.name]["internal_cert_time"]
+        ), f"internal cert for {unit.name} was not updated."
+
+        # Once the certificate requests are processed and updated the mongos.service should be
+        # restarted
+        assert (
+            new_mongos_service_time > original_tls_info[unit.name]["mongos_service"]
+        ), f"mongos service for {unit.name} was not restarted."
+
+    # Verify that TLS is functioning on all units.
+    for unit in ops_test.model.applications[MONGOS_APP_NAME].units:
+        check_mongos_tls_enabled(ops_test)