Merge branch '6/edge' into shard-removal-test

tests/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/helpers.py

@@ -1,7 +1,9 @@
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
 
+import json
 from pathlib import Path
+from typing import Dict, Optional
 
 import ops
 import yaml
@@ -100,3 +102,90 @@ async def set_password(
     )
     action = await action.wait()
     return action.results
+
+
+async def get_application_relation_data(
+    ops_test: OpsTest,
+    application_name: str,
+    relation_name: str,
+    key: str,
+    relation_id: str = None,
+    relation_alias: str = None,
+) -> Optional[str]:
+    """Get relation data for an application.
+
+    Args:
+        ops_test: The ops test framework instance
+        application_name: The name of the application
+        relation_name: name of the relation to get connection data from
+        key: key of data to be retrieved
+        relation_id: id of the relation to get connection data from
+        relation_alias: alias of the relation (like a connection name)
+            to get connection data from
+    Returns:
+        the that that was requested or None
+            if no data in the relation
+    Raises:
+        ValueError if it's not possible to get application unit data
+            or if there is no data for the particular relation endpoint
+            and/or alias.
+    """
+    unit_name = f"{application_name}/0"
+    raw_data = (await ops_test.juju("show-unit", unit_name))[1]
+
+    if not raw_data:
+        raise ValueError(f"no unit info could be grabbed for {unit_name}")
+    data = yaml.safe_load(raw_data)
+
+    # Filter the data based on the relation name.
+    relation_data = [v for v in data[unit_name]["relation-info"] if v["endpoint"] == relation_name]
+
+    if relation_id:
+        # Filter the data based on the relation id.
+        relation_data = [v for v in relation_data if v["relation-id"] == relation_id]
+
+    if relation_alias:
+        # Filter the data based on the cluster/relation alias.
+        relation_data = [
+            v
+            for v in relation_data
+            if json.loads(v["application-data"]["data"])["alias"] == relation_alias
+        ]
+
+    if len(relation_data) == 0:
+        raise ValueError(
+            f"no relation data could be grabbed on relation with endpoint {relation_name} and alias {relation_alias}"
+        )
+
+    return relation_data[0]["application-data"].get(key)
+
+
+async def get_secret_id(ops_test, app_or_unit: Optional[str] = None) -> str:
+    """Retrieve secert ID for an app or unit."""
+    complete_command = "list-secrets"
+
+    prefix = ""
+    if app_or_unit:
+        if app_or_unit[-1].isdigit():
+            # it's a unit
+            app_or_unit = "-".join(app_or_unit.split("/"))
+            prefix = "unit-"
+        else:
+            prefix = "application-"
+        complete_command += f" --owner {prefix}{app_or_unit}"
+
+    _, stdout, _ = await ops_test.juju(*complete_command.split())
+    output_lines_split = [line.split() for line in stdout.split("\n")]
+    if app_or_unit:
+        return [line[0] for line in output_lines_split if app_or_unit in line][0]
+    else:
+        return output_lines_split[1][0]
+
+
+async def get_secret_content(ops_test, secret_id) -> Dict[str, str]:
+    """Retrieve contents of a Juju Secret."""
+    secret_id = secret_id.split("/")[-1]
+    complete_command = f"show-secret {secret_id} --reveal --format=json"
+    _, stdout, _ = await ops_test.juju(*complete_command.split())
+    data = json.loads(stdout)
+    return data[secret_id]["content"]["Data"]

tests/integration/relation_tests/new_relations/helpers.py

@@ -1,68 +1,13 @@
 #!/usr/bin/env python3
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
+
 import json
-from typing import Optional
 
-import yaml
 from pytest_operator.plugin import OpsTest
 from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed
 
-
-async def get_application_relation_data(
-    ops_test: OpsTest,
-    application_name: str,
-    relation_name: str,
-    key: str,
-    relation_id: str = None,
-    relation_alias: str = None,
-) -> Optional[str]:
-    """Get relation data for an application.
-
-    Args:
-        ops_test: The ops test framework instance
-        application_name: The name of the application
-        relation_name: name of the relation to get connection data from
-        key: key of data to be retrieved
-        relation_id: id of the relation to get connection data from
-        relation_alias: alias of the relation (like a connection name)
-            to get connection data from
-    Returns:
-        the that that was requested or None
-            if no data in the relation
-    Raises:
-        ValueError if it's not possible to get application unit data
-            or if there is no data for the particular relation endpoint
-            and/or alias.
-    """
-    unit_name = f"{application_name}/0"
-    raw_data = (await ops_test.juju("show-unit", unit_name))[1]
-
-    if not raw_data:
-        raise ValueError(f"no unit info could be grabbed for {unit_name}")
-    data = yaml.safe_load(raw_data)
-
-    # Filter the data based on the relation name.
-    relation_data = [v for v in data[unit_name]["relation-info"] if v["endpoint"] == relation_name]
-
-    if relation_id:
-        # Filter the data based on the relation id.
-        relation_data = [v for v in relation_data if v["relation-id"] == relation_id]
-
-    if relation_alias:
-        # Filter the data based on the cluster/relation alias.
-        relation_data = [
-            v
-            for v in relation_data
-            if json.loads(v["application-data"]["data"])["alias"] == relation_alias
-        ]
-
-    if len(relation_data) == 0:
-        raise ValueError(
-            f"no relation data could be grabbed on relation with endpoint {relation_name} and alias {relation_alias}"
-        )
-
-    return relation_data[0]["application-data"].get(key)
+from tests.integration.helpers import get_application_relation_data
 
 
 async def verify_application_data(

tests/integration/sharding_tests/test_sharding.py

@@ -4,6 +4,7 @@
 import pytest
 from pytest_operator.plugin import OpsTest
 
+<<<<<<< HEAD
 from .helpers import (
     generate_mongodb_client,
     get_cluster_shards,
@@ -15,14 +16,24 @@
 SHARD_ONE_APP_NAME = "shard-one"
 SHARD_TWO_APP_NAME = "shard-two"
 SHARD_THREE_APP_NAME = "shard-three"
+=======
+from .helpers import generate_mongodb_client, verify_data_mongodb, write_data_to_mongodb
+
+SHARD_ONE_APP_NAME = "shard-one"
+SHARD_TWO_APP_NAME = "shard-two"
+>>>>>>> 6/edge
 CONFIG_SERVER_APP_NAME = "config-server-one"
 SHARD_REL_NAME = "sharding"
 CONFIG_SERVER_REL_NAME = "config-server"
 MONGODB_KEYFILE_PATH = "/var/snap/charmed-mongodb/current/etc/mongod/keyFile"
+<<<<<<< HEAD
 # for now we have a large timeout due to the slow drainage of the `config.system.sessions`
 # collection. More info here:
 # https://stackoverflow.com/questions/77364840/mongodb-slow-chunk-migration-for-collection-config-system-sessions-with-remov
 TIMEOUT = 30 * 60
+=======
+TIMEOUT = 15 * 60
+>>>>>>> 6/edge
 
 
 @pytest.mark.abort_on_fail
@@ -31,15 +42,26 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
     my_charm = await ops_test.build_charm(".")
     await ops_test.model.deploy(
         my_charm,
+<<<<<<< HEAD
         num_units=1,
+=======
+        num_units=2,
+>>>>>>> 6/edge
         config={"role": "config-server"},
         application_name=CONFIG_SERVER_APP_NAME,
     )
     await ops_test.model.deploy(
+<<<<<<< HEAD
         my_charm, num_units=1, config={"role": "shard"}, application_name=SHARD_ONE_APP_NAME
     )
     await ops_test.model.deploy(
         my_charm, num_units=1, config={"role": "shard"}, application_name=SHARD_TWO_APP_NAME
+=======
+        my_charm, num_units=2, config={"role": "shard"}, application_name=SHARD_ONE_APP_NAME
+    )
+    await ops_test.model.deploy(
+        my_charm, num_units=2, config={"role": "shard"}, application_name=SHARD_TWO_APP_NAME
+>>>>>>> 6/edge
     )
 
     async with ops_test.fast_forward():
@@ -48,7 +70,10 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
             idle_period=20,
             raise_on_blocked=False,
             timeout=TIMEOUT,
+<<<<<<< HEAD
             raise_on_error=False,  # checks on snaps can cause errors.
+=======
+>>>>>>> 6/edge
         )
 
     # TODO Future PR: assert that CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME
@@ -73,6 +98,7 @@ async def test_cluster_active(ops_test: OpsTest) -> None:
             idle_period=20,
             status="active",
             timeout=TIMEOUT,
+<<<<<<< HEAD
             raise_on_error=False,  # checks on snaps can cause errors.
         )
 
@@ -86,11 +112,18 @@ async def test_cluster_active(ops_test: OpsTest) -> None:
         expected_shard_names
     ), "Config server did not process config properly"
 
+=======
+        )
+
+>>>>>>> 6/edge
     # TODO Future PR: assert that CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME
     # have the correct active statuses.
 
 
+<<<<<<< HEAD
 @pytest.mark.abort_on_fail
+=======
+>>>>>>> 6/edge
 async def test_sharding(ops_test: OpsTest) -> None:
     """Tests writing data to mongos gets propagated to shards."""
     # write data to mongos on both shards.
@@ -141,6 +174,7 @@ async def test_sharding(ops_test: OpsTest) -> None:
         value="pegasus",
     )
     assert has_correct_data, "data not written to shard-two"
+<<<<<<< HEAD
 
 
 async def test_shard_removal(ops_test: OpsTest) -> None:
@@ -299,3 +333,5 @@ async def test_unconventual_shard_removal(ops_test: OpsTest):
     assert set(databases_on_shard) == set(
         expected_databases_on_shard
     ), "Not all databases on final shard"
+=======
+>>>>>>> 6/edge

tests/integration/tls_tests/helpers.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
+import logging
 from datetime import datetime
 
 import ops
@@ -20,6 +21,8 @@
 INTERNAL_CERT_PATH = f"{MONGOD_CONF_DIR}/internal-ca.crt"
 EXTERNAL_PEM_PATH = f"{MONGOD_CONF_DIR}/external-cert.pem"
 
+logger = logging.getLogger(__name__)
+
 
 class ProcessError(Exception):
     """Raised when a process fails."""
@@ -110,3 +113,22 @@ def process_systemctl_time(systemctl_output):
     time_as_str = "T".join(systemctl_output.split("=")[1].split(" ")[1:3])
     d = datetime.strptime(time_as_str, "%Y-%m-%dT%H:%M:%S")
     return d
+
+
+async def scp_file_preserve_ctime(ops_test: OpsTest, unit_name: str, path: str) -> int:
+    """Returns the unix timestamp of when a file was created on a specified unit."""
+    # Retrieving the file
+    filename = path.split("/")[-1]
+    complete_command = f"scp --container mongod {unit_name}:{path} {filename}"
+    return_code, scp_output, stderr = await ops_test.juju(*complete_command.split())
+
+    if return_code != 0:
+        logger.error(stderr)
+        raise ProcessError(
+            "Expected command %s to succeed instead it failed: %s; %s",
+            complete_command,
+            return_code,
+            stderr,
+        )
+
+    return f"{filename}"