From 6f06a482c79795570e4a1818ed64cbcad58c4c59 Mon Sep 17 00:00:00 2001 From: Mia Altieri <32723809+MiaAltieri@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:14:49 +0200 Subject: [PATCH 1/2] [DPE-2777] Basic sharding int test (#279) ## Issue No integration tests for sainity check ## Solution Add a sainty check test. i.e. deploy cluster, write data to mongos, read data from shards --- .github/workflows/ci.yaml | 1 + tests/integration/sharding_tests/helpers.py | 45 +++++++ .../sharding_tests/test_sharding.py | 120 ++++++++++++++++++ .../test_sharding_components.py | 84 ------------ tox.ini | 15 +++ 5 files changed, 181 insertions(+), 84 deletions(-) create mode 100644 tests/integration/sharding_tests/helpers.py create mode 100644 tests/integration/sharding_tests/test_sharding.py delete mode 100644 tests/integration/sharding_tests/test_sharding_components.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2670a0757..3c0bdab19 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,6 +81,7 @@ jobs: - tls-integration - backup-integration - metric-integration + - sharding-integration name: ${{ matrix.tox-environments }} needs: - lint diff --git a/tests/integration/sharding_tests/helpers.py b/tests/integration/sharding_tests/helpers.py new file mode 100644 index 000000000..868d45b1e --- /dev/null +++ b/tests/integration/sharding_tests/helpers.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +from urllib.parse import quote_plus + +from pymongo import MongoClient +from pytest_operator.plugin import OpsTest + +from ..helpers import get_password + +MONGOS_PORT = 27018 +MONGOD_PORT = 27017 + + +async def generate_mongodb_client(ops_test: OpsTest, app_name: str, mongos: bool): + """Returns a MongoDB client for mongos/mongod.""" + hosts = [unit.public_address for unit in ops_test.model.applications[app_name].units] + password = await get_password(ops_test, app_name) + port = MONGOS_PORT if mongos else MONGOD_PORT + hosts = [f"{host}:{port}" for host in hosts] + hosts = ",".join(hosts) + auth_source = "" + database = "admin" + + return MongoClient( + f"mongodb://operator:" + f"{quote_plus(password)}@" + f"{hosts}/{quote_plus(database)}?" + f"{auth_source}" + ) + + +def write_data_to_mongodb(client, db_name, coll_name, content) -> None: + """Writes data to the provided collection and database.""" + db = client[db_name] + horses_collection = db[coll_name] + horses_collection.insert_one(content) + + +def verify_data_mongodb(client, db_name, coll_name, key, value) -> bool: + """Checks a key/value pair for a provided collection and database.""" + db = client[db_name] + test_collection = db[coll_name] + query = test_collection.find({}, {key: 1}) + return query[0][key] == value diff --git a/tests/integration/sharding_tests/test_sharding.py b/tests/integration/sharding_tests/test_sharding.py new file mode 100644 index 000000000..d22166e86 --- /dev/null +++ b/tests/integration/sharding_tests/test_sharding.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +import pytest +from pytest_operator.plugin import OpsTest + +from .helpers import generate_mongodb_client, verify_data_mongodb, write_data_to_mongodb + +SHARD_ONE_APP_NAME = "shard-one" +SHARD_TWO_APP_NAME = "shard-two" +CONFIG_SERVER_APP_NAME = "config-server-one" +SHARD_REL_NAME = "sharding" +CONFIG_SERVER_REL_NAME = "config-server" +MONGODB_KEYFILE_PATH = "/var/snap/charmed-mongodb/current/etc/mongod/keyFile" +TIMEOUT = 15 * 60 + + +@pytest.mark.abort_on_fail +async def test_build_and_deploy(ops_test: OpsTest) -> None: + """Build and deploy a sharded cluster.""" + my_charm = await ops_test.build_charm(".") + await ops_test.model.deploy( + my_charm, + num_units=2, + config={"role": "config-server"}, + application_name=CONFIG_SERVER_APP_NAME, + ) + await ops_test.model.deploy( + my_charm, num_units=2, config={"role": "shard"}, application_name=SHARD_ONE_APP_NAME + ) + await ops_test.model.deploy( + my_charm, num_units=2, config={"role": "shard"}, application_name=SHARD_TWO_APP_NAME + ) + + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle( + apps=[CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME], + idle_period=20, + raise_on_blocked=False, + timeout=TIMEOUT, + ) + + # TODO Future PR: assert that CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME + # are blocked waiting for relaitons + + +@pytest.mark.abort_on_fail +async def test_cluster_active(ops_test: OpsTest) -> None: + """Tests the integration of cluster components works without error.""" + await ops_test.model.integrate( + f"{SHARD_ONE_APP_NAME}:{SHARD_REL_NAME}", + f"{CONFIG_SERVER_APP_NAME}:{CONFIG_SERVER_REL_NAME}", + ) + await ops_test.model.integrate( + f"{SHARD_TWO_APP_NAME}:{SHARD_REL_NAME}", + f"{CONFIG_SERVER_APP_NAME}:{CONFIG_SERVER_REL_NAME}", + ) + + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle( + apps=[CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME], + idle_period=20, + status="active", + timeout=TIMEOUT, + ) + + # TODO Future PR: assert that CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME + # have the correct active statuses. + + +async def test_sharding(ops_test: OpsTest) -> None: + """Tests writing data to mongos gets propagated to shards.""" + # write data to mongos on both shards. + mongos_client = await generate_mongodb_client( + ops_test, app_name=CONFIG_SERVER_APP_NAME, mongos=True + ) + + # write data to shard one + write_data_to_mongodb( + mongos_client, + db_name="animals_database_1", + coll_name="horses", + content={"horse-breed": "unicorn", "real": True}, + ) + mongos_client.admin.command("movePrimary", "animals_database_1", to=SHARD_ONE_APP_NAME) + + # write data to shard two + write_data_to_mongodb( + mongos_client, + db_name="animals_database_2", + coll_name="horses", + content={"horse-breed": "pegasus", "real": True}, + ) + mongos_client.admin.command("movePrimary", "animals_database_2", to=SHARD_TWO_APP_NAME) + + # log into shard 1 verify data + shard_one_client = await generate_mongodb_client( + ops_test, app_name=SHARD_ONE_APP_NAME, mongos=False + ) + has_correct_data = verify_data_mongodb( + shard_one_client, + db_name="animals_database_1", + coll_name="horses", + key="horse-breed", + value="unicorn", + ) + assert has_correct_data, "data not written to shard-one" + + # log into shard 2 verify data + shard_two_client = await generate_mongodb_client( + ops_test, app_name=SHARD_TWO_APP_NAME, mongos=False + ) + has_correct_data = verify_data_mongodb( + shard_two_client, + db_name="animals_database_2", + coll_name="horses", + key="horse-breed", + value="pegasus", + ) + assert has_correct_data, "data not written to shard-two" diff --git a/tests/integration/sharding_tests/test_sharding_components.py b/tests/integration/sharding_tests/test_sharding_components.py deleted file mode 100644 index e4503f5b9..000000000 --- a/tests/integration/sharding_tests/test_sharding_components.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -import pytest -from pytest_operator.plugin import OpsTest - -from ..helpers import get_password - -SHARD_ONE_APP_NAME = "shard-one" -CONFIG_SERVER_APP_NAME = "config-server-one" -SHARD_REL_NAME = "sharding" -CONFIG_SERVER_REL_NAME = "config-server" -MONGODB_KEYFILE_PATH = "/var/snap/charmed-mongodb/current/etc/mongod/keyFile" - -""" -Integration tests are not a requirement for our release goal of POC Sharding. However they are -useful for automating tests that developers must do by hand. This file currently exists to help -running tests during the development of sharding. - -TODO Future tests: -- shard can only relate to one config server -- shard cannot change passwords -""" - - -@pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest) -> None: - """Build and deploy a sharded cluster.""" - async with ops_test.fast_forward(): - my_charm = await ops_test.build_charm(".") - await ops_test.model.deploy( - my_charm, - num_units=3, - config={"role": "config-server"}, - application_name=CONFIG_SERVER_APP_NAME, - ) - await ops_test.model.deploy( - my_charm, num_units=3, config={"role": "shard"}, application_name=SHARD_ONE_APP_NAME - ) - await ops_test.model.wait_for_idle( - apps=[CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME], status="active" - ) - - await ops_test.model.integrate( - f"{SHARD_ONE_APP_NAME}:{SHARD_REL_NAME}", - f"{CONFIG_SERVER_APP_NAME}:{CONFIG_SERVER_REL_NAME}", - ) - await ops_test.model.wait_for_idle( - apps=[CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME], status="active" - ) - - -async def test_shared_operator_password(ops_test: OpsTest) -> None: - """Verify sharded components have the same passwords.""" - shard_password = await get_password(ops_test, SHARD_ONE_APP_NAME) - config_password = await get_password(ops_test, CONFIG_SERVER_APP_NAME) - assert ( - shard_password == config_password - ), "sharding components do not have the same operator password." - - -async def test_shared_keyfile(ops_test: OpsTest) -> None: - """Verify sharded components have the same keyfile contents.""" - config_unit = ops_test.model.applications[CONFIG_SERVER_APP_NAME].units[0] - config_key_file = await get_keyfile_contents(config_unit) - - shard_unit = ops_test.model.applications[SHARD_ONE_APP_NAME].units[0] - shard_key_file = await get_keyfile_contents(shard_unit) - - assert config_key_file == shard_key_file, "shards and config server using different keyfiles" - - -async def get_keyfile_contents(ops_test: OpsTest, unit) -> str: - cat_cmd = f"exec --unit {unit.name} -- cat {MONGODB_KEYFILE_PATH}" - return_code, output, _ = await ops_test.juju(*cat_cmd.split()) - - if return_code != 0: - raise ProcessError( - f"Expected cat command {cat_cmd} to succeed instead it failed: {return_code}" - ) - - -class ProcessError(Exception): - """Raised when a process fails.""" diff --git a/tox.ini b/tox.ini index c89fdca5b..8c038773f 100644 --- a/tox.ini +++ b/tox.ini @@ -177,6 +177,21 @@ deps = commands = pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/metrics_tests/test_metrics.py +[testenv:sharding-integration] +description = Run sharding integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS +deps = + pytest + juju==3.2.0.1 + pytest-mock + pytest-operator + -r {tox_root}/requirements.txt +commands = + pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/sharding_tests/test_sharding.py + [testenv:integration] description = Run all integration tests From b62aba93a31721f178e9be28662cee1b9024269d Mon Sep 17 00:00:00 2001 From: Judit Novak Date: Wed, 25 Oct 2023 10:00:44 +0200 Subject: [PATCH 2/2] [DPE-2790] Extended TLS testing (6/edge) (#281) ## Issue As already done for K8s: https://github.com/canonical/mongodb-k8s-operator/pull/206 ## Solution --- tests/__init__.py | 2 + tests/integration/helpers.py | 89 +++++++++++++++++++ .../relation_tests/new_relations/helpers.py | 59 +----------- tests/integration/tls_tests/helpers.py | 22 +++++ tests/integration/tls_tests/test_tls.py | 63 +++++++++++++ 5 files changed, 178 insertions(+), 57 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..db3bfe1a6 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 755f6e464..3ce39568f 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,7 +1,9 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. +import json from pathlib import Path +from typing import Dict, Optional import ops import yaml @@ -100,3 +102,90 @@ async def set_password( ) action = await action.wait() return action.results + + +async def get_application_relation_data( + ops_test: OpsTest, + application_name: str, + relation_name: str, + key: str, + relation_id: str = None, + relation_alias: str = None, +) -> Optional[str]: + """Get relation data for an application. + + Args: + ops_test: The ops test framework instance + application_name: The name of the application + relation_name: name of the relation to get connection data from + key: key of data to be retrieved + relation_id: id of the relation to get connection data from + relation_alias: alias of the relation (like a connection name) + to get connection data from + Returns: + the that that was requested or None + if no data in the relation + Raises: + ValueError if it's not possible to get application unit data + or if there is no data for the particular relation endpoint + and/or alias. + """ + unit_name = f"{application_name}/0" + raw_data = (await ops_test.juju("show-unit", unit_name))[1] + + if not raw_data: + raise ValueError(f"no unit info could be grabbed for {unit_name}") + data = yaml.safe_load(raw_data) + + # Filter the data based on the relation name. + relation_data = [v for v in data[unit_name]["relation-info"] if v["endpoint"] == relation_name] + + if relation_id: + # Filter the data based on the relation id. + relation_data = [v for v in relation_data if v["relation-id"] == relation_id] + + if relation_alias: + # Filter the data based on the cluster/relation alias. + relation_data = [ + v + for v in relation_data + if json.loads(v["application-data"]["data"])["alias"] == relation_alias + ] + + if len(relation_data) == 0: + raise ValueError( + f"no relation data could be grabbed on relation with endpoint {relation_name} and alias {relation_alias}" + ) + + return relation_data[0]["application-data"].get(key) + + +async def get_secret_id(ops_test, app_or_unit: Optional[str] = None) -> str: + """Retrieve secert ID for an app or unit.""" + complete_command = "list-secrets" + + prefix = "" + if app_or_unit: + if app_or_unit[-1].isdigit(): + # it's a unit + app_or_unit = "-".join(app_or_unit.split("/")) + prefix = "unit-" + else: + prefix = "application-" + complete_command += f" --owner {prefix}{app_or_unit}" + + _, stdout, _ = await ops_test.juju(*complete_command.split()) + output_lines_split = [line.split() for line in stdout.split("\n")] + if app_or_unit: + return [line[0] for line in output_lines_split if app_or_unit in line][0] + else: + return output_lines_split[1][0] + + +async def get_secret_content(ops_test, secret_id) -> Dict[str, str]: + """Retrieve contents of a Juju Secret.""" + secret_id = secret_id.split("/")[-1] + complete_command = f"show-secret {secret_id} --reveal --format=json" + _, stdout, _ = await ops_test.juju(*complete_command.split()) + data = json.loads(stdout) + return data[secret_id]["content"]["Data"] diff --git a/tests/integration/relation_tests/new_relations/helpers.py b/tests/integration/relation_tests/new_relations/helpers.py index 162c9d8f1..dcaf62bc8 100644 --- a/tests/integration/relation_tests/new_relations/helpers.py +++ b/tests/integration/relation_tests/new_relations/helpers.py @@ -1,68 +1,13 @@ #!/usr/bin/env python3 # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. + import json -from typing import Optional -import yaml from pytest_operator.plugin import OpsTest from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed - -async def get_application_relation_data( - ops_test: OpsTest, - application_name: str, - relation_name: str, - key: str, - relation_id: str = None, - relation_alias: str = None, -) -> Optional[str]: - """Get relation data for an application. - - Args: - ops_test: The ops test framework instance - application_name: The name of the application - relation_name: name of the relation to get connection data from - key: key of data to be retrieved - relation_id: id of the relation to get connection data from - relation_alias: alias of the relation (like a connection name) - to get connection data from - Returns: - the that that was requested or None - if no data in the relation - Raises: - ValueError if it's not possible to get application unit data - or if there is no data for the particular relation endpoint - and/or alias. - """ - unit_name = f"{application_name}/0" - raw_data = (await ops_test.juju("show-unit", unit_name))[1] - - if not raw_data: - raise ValueError(f"no unit info could be grabbed for {unit_name}") - data = yaml.safe_load(raw_data) - - # Filter the data based on the relation name. - relation_data = [v for v in data[unit_name]["relation-info"] if v["endpoint"] == relation_name] - - if relation_id: - # Filter the data based on the relation id. - relation_data = [v for v in relation_data if v["relation-id"] == relation_id] - - if relation_alias: - # Filter the data based on the cluster/relation alias. - relation_data = [ - v - for v in relation_data - if json.loads(v["application-data"]["data"])["alias"] == relation_alias - ] - - if len(relation_data) == 0: - raise ValueError( - f"no relation data could be grabbed on relation with endpoint {relation_name} and alias {relation_alias}" - ) - - return relation_data[0]["application-data"].get(key) +from tests.integration.helpers import get_application_relation_data async def verify_application_data( diff --git a/tests/integration/tls_tests/helpers.py b/tests/integration/tls_tests/helpers.py index a87f167bf..a3fd4065d 100644 --- a/tests/integration/tls_tests/helpers.py +++ b/tests/integration/tls_tests/helpers.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. +import logging from datetime import datetime import ops @@ -20,6 +21,8 @@ INTERNAL_CERT_PATH = f"{MONGOD_CONF_DIR}/internal-ca.crt" EXTERNAL_PEM_PATH = f"{MONGOD_CONF_DIR}/external-cert.pem" +logger = logging.getLogger(__name__) + class ProcessError(Exception): """Raised when a process fails.""" @@ -110,3 +113,22 @@ def process_systemctl_time(systemctl_output): time_as_str = "T".join(systemctl_output.split("=")[1].split(" ")[1:3]) d = datetime.strptime(time_as_str, "%Y-%m-%dT%H:%M:%S") return d + + +async def scp_file_preserve_ctime(ops_test: OpsTest, unit_name: str, path: str) -> int: + """Returns the unix timestamp of when a file was created on a specified unit.""" + # Retrieving the file + filename = path.split("/")[-1] + complete_command = f"scp --container mongod {unit_name}:{path} {filename}" + return_code, scp_output, stderr = await ops_test.juju(*complete_command.split()) + + if return_code != 0: + logger.error(stderr) + raise ProcessError( + "Expected command %s to succeed instead it failed: %s; %s", + complete_command, + return_code, + stderr, + ) + + return f"{filename}" diff --git a/tests/integration/tls_tests/test_tls.py b/tests/integration/tls_tests/test_tls.py index db67d09e2..9b932c606 100644 --- a/tests/integration/tls_tests/test_tls.py +++ b/tests/integration/tls_tests/test_tls.py @@ -2,24 +2,82 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. +import json + import pytest +from ops import Unit from pytest_operator.plugin import OpsTest +from ..helpers import get_application_relation_data, get_secret_content, get_secret_id from .helpers import ( EXTERNAL_CERT_PATH, INTERNAL_CERT_PATH, check_tls, + scp_file_preserve_ctime, time_file_created, time_process_started, ) MONGO_COMMON_DIR = "/var/snap/charmed-mongodb/common" TLS_CERTIFICATES_APP_NAME = "tls-certificates-operator" +TLS_RELATION_NAME = "certificates" DATABASE_APP_NAME = "mongodb" TLS_TEST_DATA = "tests/integration/tls_tests/data" DB_SERVICE = "snap.charmed-mongodb.mongod.service" +async def check_certs_correctly_distributed(ops_test: OpsTest, unit: Unit) -> None: + """Comparing expected vs distributed certificates. + + Verifying certificates downloaded on the charm against the ones distributed by the TLS operator + """ + app_secret_id = await get_secret_id(ops_test, DATABASE_APP_NAME) + unit_secret_id = await get_secret_id(ops_test, unit.name) + app_secret_content = await get_secret_content(ops_test, app_secret_id) + unit_secret_content = await get_secret_content(ops_test, unit_secret_id) + app_current_crt = app_secret_content["csr-secret"] + unit_current_crt = unit_secret_content["csr-secret"] + + # Get the values for certs from the relation, as provided by TLS Charm + certificates_raw_data = await get_application_relation_data( + ops_test, DATABASE_APP_NAME, TLS_RELATION_NAME, "certificates" + ) + certificates_data = json.loads(certificates_raw_data) + + external_item = [ + data + for data in certificates_data + if data["certificate_signing_request"].rstrip() == unit_current_crt.rstrip() + ][0] + internal_item = [ + data + for data in certificates_data + if data["certificate_signing_request"].rstrip() == app_current_crt.rstrip() + ][0] + + # Get a local copy of the external cert + external_copy_path = await scp_file_preserve_ctime(ops_test, unit.name, EXTERNAL_CERT_PATH) + + # Get the external cert value from the relation + relation_external_cert = "\n".join(external_item["chain"]) + + # CHECK: Compare if they are the same + with open(external_copy_path) as f: + external_contents_file = f.read() + assert relation_external_cert == external_contents_file + + # Get a local copy of the internal cert + internal_copy_path = await scp_file_preserve_ctime(ops_test, unit.name, INTERNAL_CERT_PATH) + + # Get the external cert value from the relation + relation_internal_cert = "\n".join(internal_item["chain"]) + + # CHECK: Compare if they are the same + with open(internal_copy_path) as f: + internal_contents_file = f.read() + assert relation_internal_cert == internal_contents_file + + @pytest.mark.abort_on_fail async def test_build_and_deploy(ops_test: OpsTest) -> None: """Build and deploy one unit of MongoDB and one unit of TLS.""" @@ -67,6 +125,7 @@ async def test_rotate_tls_key(ops_test: OpsTest) -> None: original_tls_times[unit.name]["mongod_service"] = await time_process_started( ops_test, unit.name, DB_SERVICE ) + check_certs_correctly_distributed(ops_test, unit) # set external and internal key using auto-generated key for each unit for unit in ops_test.model.applications[DATABASE_APP_NAME].units: @@ -87,6 +146,8 @@ async def test_rotate_tls_key(ops_test: OpsTest) -> None: new_internal_cert_time = await time_file_created(ops_test, unit.name, INTERNAL_CERT_PATH) new_mongod_service_time = await time_process_started(ops_test, unit.name, DB_SERVICE) + check_certs_correctly_distributed(ops_test, unit) + assert ( new_external_cert_time > original_tls_times[unit.name]["external_cert"] ), f"external cert for {unit.name} was not updated." @@ -165,6 +226,8 @@ async def test_set_tls_key(ops_test: OpsTest) -> None: new_internal_cert_time = await time_file_created(ops_test, unit.name, INTERNAL_CERT_PATH) new_mongod_service_time = await time_process_started(ops_test, unit.name, DB_SERVICE) + check_certs_correctly_distributed(ops_test, unit) + assert ( new_external_cert_time > original_tls_times[unit.name]["external_cert"] ), f"external cert for {unit.name} was not updated."