From 49658372628084b3b931edc0f5af1391ec1f7f6c Mon Sep 17 00:00:00 2001 From: Dmitry Ratushnyy Date: Sun, 7 Jan 2024 09:35:03 +0000 Subject: [PATCH] Fixed tests --- tests/integration/ha_tests/helpers.py | 6 ++-- tests/integration/ha_tests/test_ha.py | 41 ++++++--------------------- tests/integration/helpers.py | 8 +++--- tests/integration/test_charm.py | 13 +++++---- 4 files changed, 23 insertions(+), 45 deletions(-) diff --git a/tests/integration/ha_tests/helpers.py b/tests/integration/ha_tests/helpers.py index efc279200..ce37572bc 100644 --- a/tests/integration/ha_tests/helpers.py +++ b/tests/integration/ha_tests/helpers.py @@ -73,7 +73,7 @@ async def fetch_replica_set_members(replica_ips: List[str], ops_test: OpsTest, a Args: replica_ips: list of ips hosting the replica set. ops_test: reference to deployment. - app: name of application which has the cluster. + app_name: name of application which has the cluster. """ # connect to replica set uri app_name = app_name or await get_app_name(ops_test) @@ -151,11 +151,11 @@ async def fetch_primary( # TODO remove duplication with common helpers -async def count_primaries(ops_test: OpsTest, app_name=None) -> int: +async def count_primaries(ops_test: OpsTest, password: str = None, app_name: str = None) -> int: """Returns the number of primaries in a replica set.""" # connect to MongoDB client app_name = app_name or await get_app_name(ops_test) - password = await get_password(ops_test, app_name) + password = password or await get_password(ops_test, app_name) replica_set_hosts = [ unit.public_address for unit in ops_test.model.applications[app_name].units ] diff --git a/tests/integration/ha_tests/test_ha.py b/tests/integration/ha_tests/test_ha.py index 340ecc2df..19db647fe 100644 --- a/tests/integration/ha_tests/test_ha.py +++ b/tests/integration/ha_tests/test_ha.py @@ -24,20 +24,14 @@ add_unit_with_storage, all_db_processes_down, clear_db_writes, -) -from .helpers import ( - count_primaries as count_primaries_ha, # TODO remove this duplication -) -from .helpers import ( + count_primaries, count_writes, cut_network_from_unit, db_step_down, fetch_replica_set_members, find_unit, get_controller_machine, -) -from .helpers import get_password as get_password_ha # TODO remove this duplication -from .helpers import ( + get_password, insert_focal_to_cluster, is_machine_reachable_from, kill_unit_process, @@ -236,7 +230,6 @@ async def test_scale_down_capablities(ops_test: OpsTest, continuous_writes) -> N assert total_expected_writes["number"] == actual_writes -@pytest.mark.skip("skip") async def test_replication_across_members(ops_test: OpsTest, continuous_writes) -> None: """Check consistency, ie write to primary, read data from secondaries.""" # first find primary, write to primary, then read from each unit @@ -244,7 +237,7 @@ async def test_replication_across_members(ops_test: OpsTest, continuous_writes) app_name = await get_app_name(ops_test) ip_addresses = [unit.public_address for unit in ops_test.model.applications[app_name].units] primary = await replica_set_primary(ip_addresses, ops_test, app_name=app_name) - password = await get_password_ha(ops_test, app_name) + password = await get_password(ops_test, app_name) secondaries = set(ip_addresses) - set([primary.public_address]) for secondary in secondaries: @@ -263,7 +256,6 @@ async def test_replication_across_members(ops_test: OpsTest, continuous_writes) assert total_expected_writes["number"] == actual_writes -@pytest.mark.skip("skip") async def test_unique_cluster_dbs(ops_test: OpsTest, continuous_writes) -> None: """Verify unique clusters do not share DBs.""" # first find primary, write to primary, @@ -280,7 +272,7 @@ async def test_unique_cluster_dbs(ops_test: OpsTest, continuous_writes) -> None: unit.public_address for unit in ops_test.model.applications[ANOTHER_DATABASE_APP_NAME].units ] - password = await get_password_ha(ops_test, app_name=ANOTHER_DATABASE_APP_NAME) + password = await get_password(ops_test, app_name=ANOTHER_DATABASE_APP_NAME) client = replica_set_client(ip_addresses, password, app_name=ANOTHER_DATABASE_APP_NAME) db = client["new-db"] test_collection = db["test_ubuntu_collection"] @@ -312,7 +304,6 @@ async def test_unique_cluster_dbs(ops_test: OpsTest, continuous_writes) -> None: assert total_expected_writes["number"] == actual_writes -@pytest.mark.skip("skip") async def test_replication_member_scaling(ops_test: OpsTest, continuous_writes) -> None: """Verify newly added and newly removed members properly replica data. @@ -335,7 +326,7 @@ async def test_replication_member_scaling(ops_test: OpsTest, continuous_writes) unit.public_address for unit in ops_test.model.applications[app_name].units ] new_member_ip = list(set(new_ip_addresses) - set(original_ip_addresses))[0] - password = await get_password_ha(ops_test, app_name) + password = await get_password(ops_test, app_name) client = MongoClient(unit_uri(new_member_ip, password, app_name), directConnection=True) # check for replicated data while retrying to give time for replica to copy over data. @@ -358,7 +349,6 @@ async def test_replication_member_scaling(ops_test: OpsTest, continuous_writes) assert total_expected_writes["number"] == actual_writes -@pytest.mark.skip("skip") async def test_kill_db_process(ops_test, continuous_writes): # locate primary unit app_name = await get_app_name(ops_test) @@ -397,12 +387,10 @@ async def test_kill_db_process(ops_test, continuous_writes): async def test_freeze_db_process(ops_test, continuous_writes): # locate primary unit - logger.error(">>>>>>>>>>>>>>>>>>> test_freeze_db_process 1") app_name = await get_app_name(ops_test) - password = await get_password_ha(ops_test, app_name) + password = await get_password(ops_test, app_name) ip_addresses = [unit.public_address for unit in ops_test.model.applications[app_name].units] primary = await replica_set_primary(ip_addresses, ops_test, app_name=app_name) - logger.error(">>>>>>>>>>>>>>>>>>> test_freeze_db_process 2") await kill_unit_process(ops_test, primary.name, kill_code="SIGSTOP", app_name=app_name) # sleep for twice the median election time @@ -412,13 +400,11 @@ async def test_freeze_db_process(ops_test, continuous_writes): new_primary = await replica_set_primary(ip_addresses, ops_test, app_name=app_name) assert new_primary.name != primary.name - logger.error(">>>>>>>>>>>>>>>>>>> test_freeze_db_process 3") # verify new writes are continuing by counting the number of writes before and after a 5 second # wait writes = await count_writes(ops_test, app_name=app_name) time.sleep(5) more_writes = await count_writes(ops_test, app_name=app_name) - logger.error(">>>>>>>>>>>>>>>>>>> test_freeze_db_process 4") # un-freeze the old primary await kill_unit_process(ops_test, primary.name, kill_code="SIGCONT", app_name=app_name) @@ -432,30 +418,26 @@ async def test_freeze_db_process(ops_test, continuous_writes): # verify all units are running under the same replset member_ips = await fetch_replica_set_members(ip_addresses, ops_test, app_name=app_name) assert set(member_ips) == set(ip_addresses), "all members not running under the same replset" - logger.error(">>>>>>>>>>>>>>>>>>> test_freeze_db_process 5") + # verify there is only one primary after un-freezing old primary assert ( - await count_primaries_ha(ops_test, password, app_name=app_name) == 1 + await count_primaries(ops_test, password=password, app_name=app_name) == 1 ), "there are more than one primary in the replica set." # verify that the old primary does not "reclaim" primary status after un-freezing old primary new_primary = await replica_set_primary(ip_addresses, ops_test, app_name=app_name) - logger.error(">>>>>>>>>>>>>>>>>>> test_freeze_db_process 6") assert new_primary.name != primary.name, "un-frozen primary should be secondary." # verify that no writes were missed. total_expected_writes = await stop_continous_writes(ops_test, app_name=app_name) actual_writes = await count_writes(ops_test, app_name=app_name) assert actual_writes == total_expected_writes["number"], "db writes missing." - logger.error(">>>>>>>>>>>>>>>>>>> test_freeze_db_process 7") # verify that old primary is up to date. assert await secondary_up_to_date( ops_test, primary.public_address, actual_writes, app_name=app_name ), "secondary not up to date with the cluster after restarting." - logger.error(">>>>>>>>>>>>>>>>>>> test_freeze_db_process 8") -@pytest.mark.skip("skip") async def test_restart_db_process(ops_test, continuous_writes, change_logging): # locate primary unit app_name = await get_app_name(ops_test) @@ -502,7 +484,6 @@ async def test_restart_db_process(ops_test, continuous_writes, change_logging): ), "secondary not up to date with the cluster after restarting." -@pytest.mark.skip("skip") async def test_full_cluster_crash(ops_test: OpsTest, continuous_writes, reset_restart_delay): app_name = await get_app_name(ops_test) @@ -553,9 +534,8 @@ async def test_full_cluster_crash(ops_test: OpsTest, continuous_writes, reset_re assert actual_writes == total_expected_writes["number"], "db writes missing." -@pytest.mark.skip("skip") async def test_full_cluster_restart(ops_test: OpsTest, continuous_writes, reset_restart_delay): - app_name = await get_app_name(ops_test, app_name=app_name) + app_name = await get_app_name(ops_test) # update all units to have a new RESTART_DELAY, Modifying the Restart delay to 3 minutes # should ensure enough time for all replicas to be down at the same time. @@ -602,7 +582,6 @@ async def test_full_cluster_restart(ops_test: OpsTest, continuous_writes, reset_ assert total_expected_writes["number"] == actual_writes, "writes to the db were missed." -@pytest.mark.skip("skip") async def test_network_cut(ops_test, continuous_writes): # locate primary unit app_name = await get_app_name(ops_test) @@ -682,7 +661,6 @@ async def test_network_cut(ops_test, continuous_writes): ), "secondary not up to date with the cluster after restarting." -@pytest.mark.skip("skip") @pytest.mark.abort_on_fail @pytest.mark.unstable async def test_scale_up_down(ops_test: OpsTest, continuous_writes): @@ -693,7 +671,6 @@ async def test_scale_up_down(ops_test: OpsTest, continuous_writes): await verify_writes(ops_test) -@pytest.mark.skip("skip") @pytest.mark.abort_on_fail @pytest.mark.unstable async def test_scale_up_down_removing_leader(ops_test: OpsTest, continuous_writes): diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index a2c901936..b6b592ed0 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -61,12 +61,12 @@ async def get_password(ops_test: OpsTest, username="operator", app_name=None) -> stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=2, max=30), ) -async def count_primaries(ops_test: OpsTest, password: str) -> int: +async def count_primaries(ops_test: OpsTest, password: str, app_name=None) -> int: """Counts the number of primaries in a replica set. Will retry counting when the number of primaries is 0 at most 5 times. """ - app_name = await get_app_name(ops_test) + app_name = app_name or await get_app_name(ops_test) number_of_primaries = 0 for unit_id in UNIT_IDS: # get unit @@ -84,9 +84,9 @@ async def count_primaries(ops_test: OpsTest, password: str) -> int: return number_of_primaries -async def find_unit(ops_test: OpsTest, leader: bool) -> ops.model.Unit: +async def find_unit(ops_test: OpsTest, leader: bool, app_name=None) -> ops.model.Unit: """Helper function identifies the a unit, based on need for leader or non-leader.""" - app_name = await get_app_name(ops_test) + app_name = app_name or await get_app_name(ops_test) ret_unit = None for unit in ops_test.model.applications[app_name].units: if await unit.is_leader_from_status() == leader: diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 932ec8d26..82ed8ff7c 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -82,17 +82,17 @@ async def test_unit_is_running_as_replica_set(ops_test: OpsTest, unit_id: int) - async def test_leader_is_primary_on_deployment(ops_test: OpsTest) -> None: """Tests that right after deployment that the primary unit is the leader.""" + app_name = await get_app_name(ops_test) # grab leader unit - leader_unit = await find_unit(ops_test, leader=True) + leader_unit = await find_unit(ops_test, leader=True, app_name=app_name) # verify that we have a leader assert leader_unit is not None, "No unit is leader" # connect to mongod - password = await get_password(ops_test) - user_app_name = await get_app_name(ops_test) + password = await get_password(ops_test, app_name=app_name) client = MongoClient( - unit_uri(leader_unit.public_address, password, user_app_name), directConnection=True + unit_uri(leader_unit.public_address, password, app_name), directConnection=True ) # verify primary status @@ -102,9 +102,10 @@ async def test_leader_is_primary_on_deployment(ops_test: OpsTest) -> None: async def test_exactly_one_primary(ops_test: OpsTest) -> None: """Tests that there is exactly one primary in the deployed units.""" + app_name = await get_app_name(ops_test) try: - password = await get_password(ops_test) - number_of_primaries = await count_primaries(ops_test, password) + password = await get_password(ops_test, app_name=app_name) + number_of_primaries = await count_primaries(ops_test, password, app_name=app_name) except RetryError: number_of_primaries = 0