Fix CI issues with snap install and race conditions in sharding test (#…

…287) ## Issue older version of data platform actions has issues with checking installed snaps there are race conditions present in the charm that cause issues with tests ## Solution update data platform actions resolve race conditions ## Other add status checks
canonical · Oct 30, 2023 · 347f777 · 347f777
1 parent cc52ce1
commit 347f777
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 16 deletions.
diff --git a/lib/charms/mongodb/v1/shards_interface.py b/lib/charms/mongodb/v1/shards_interface.py
@@ -386,6 +386,7 @@ def _on_relation_changed(self, event):
             logger.error(
                 "Shard could not be added to config server, failed to set operator password."
             )
+            event.defer()
             return
 
         self.charm.app_peer_data["added_to_cluster"] = json.dumps(True)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
@@ -6,11 +6,9 @@
 import os
 from pathlib import Path
 
-import pytest
 from pytest_operator.plugin import OpsTest
 
 
-@pytest.fixture(scope="module")
 def ops_test(ops_test: OpsTest) -> OpsTest:
     if os.environ.get("CI") == "true":
         # Running in GitHub Actions; skip build step

diff --git a/tests/integration/sharding_tests/test_sharding.py b/tests/integration/sharding_tests/test_sharding.py
@@ -1,18 +1,24 @@
 #!/usr/bin/env python3
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
+import asyncio
+
 import pytest
 from pytest_operator.plugin import OpsTest
 
 from .helpers import generate_mongodb_client, verify_data_mongodb, write_data_to_mongodb
 
 SHARD_ONE_APP_NAME = "shard-one"
 SHARD_TWO_APP_NAME = "shard-two"
+SHARD_APPS = [SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME]
 CONFIG_SERVER_APP_NAME = "config-server-one"
 SHARD_REL_NAME = "sharding"
 CONFIG_SERVER_REL_NAME = "config-server"
 MONGODB_KEYFILE_PATH = "/var/snap/charmed-mongodb/current/etc/mongod/keyFile"
-TIMEOUT = 15 * 60
+# for now we have a large timeout due to the slow drainage of the `config.system.sessions`
+# collection. More info here:
+# https://stackoverflow.com/questions/77364840/mongodb-slow-chunk-migration-for-collection-config-system-sessions-with-remov
+TIMEOUT = 30 * 60
 
 
 @pytest.mark.abort_on_fail
@@ -40,8 +46,26 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
             timeout=TIMEOUT,
         )
 
-    # TODO Future PR: assert that CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME
-    # are blocked waiting for relaitons
+    # verify that Charmed MongoDB is blocked and reports incorrect credentials
+    await asyncio.gather(
+        ops_test.model.wait_for_idle(
+            apps=[CONFIG_SERVER_APP_NAME],
+            status="active",
+            idle_period=20,
+            timeout=TIMEOUT,
+        ),
+        ops_test.model.wait_for_idle(
+            apps=[SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME],
+            status="blocked",
+            idle_period=20,
+            timeout=TIMEOUT,
+        ),
+    )
+
+    # TODO Future PR: assert statuses for config-server
+    for shard_app_name in SHARD_APPS:
+        shard_unit = ops_test.model.applications[shard_app_name].units[0]
+        assert shard_unit.workload_status_message == "missing relation to config server"
 
 
 @pytest.mark.abort_on_fail
@@ -64,8 +88,13 @@ async def test_cluster_active(ops_test: OpsTest) -> None:
             timeout=TIMEOUT,
         )
 
-    # TODO Future PR: assert that CONFIG_SERVER_APP_NAME, SHARD_ONE_APP_NAME, SHARD_TWO_APP_NAME
-    # have the correct active statuses.
+    # TODO Future PR: assert statuses for config-server
+    for shard_app_name in SHARD_APPS:
+        shard_unit = ops_test.model.applications[shard_app_name].units[0]
+        assert (
+            shard_unit.workload_status_message
+            == f"Shard connected to config-server: {CONFIG_SERVER_APP_NAME}"
+        )
 
 
 async def test_sharding(ops_test: OpsTest) -> None:

diff --git a/tox.ini b/tox.ini
@@ -72,12 +72,12 @@ description = Run charm integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
 deps =
     pytest
     juju==3.2.0.1
     pytest-operator
     pytest-mock
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/test_charm.py
@@ -87,12 +87,12 @@ description = Run high availability integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
 deps =
     pytest
     juju==3.2.0.1
     pytest-mock
     pytest-operator
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -vvv --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/ha_tests/test_ha.py
@@ -102,12 +102,12 @@ description = Run new relation integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
 deps =
     pytest
     juju==3.2.0.1
     pytest-mock
     pytest-operator
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/relation_tests/new_relations/test_charm_relations.py
@@ -117,12 +117,12 @@ description = Run legacy relation integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
 deps =
     pytest
     juju==3.2.0.1
     pytest-mock
     pytest-operator
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/relation_tests/legacy_relations/test_charm_legacy_relations.py
@@ -132,12 +132,12 @@ description = Run tls integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
 deps =
     pytest
     juju==3.2.0.1
     pytest-mock
     pytest-operator
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/tls_tests/test_tls.py
@@ -148,7 +148,6 @@ description = Run backup integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
     AWS_ACCESS_KEY
     AWS_SECRET_KEY
     GCP_ACCESS_KEY
@@ -158,6 +157,7 @@ deps =
     juju==3.2.0.1
     pytest-mock
     pytest-operator
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/backup_tests/test_backups.py
@@ -167,12 +167,12 @@ description = Run metrics integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
 deps =
     pytest
     juju==3.2.0.1
     pytest-mock
     pytest-operator
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/metrics_tests/test_metrics.py
@@ -182,12 +182,12 @@ description = Run sharding integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
 deps =
     pytest
     juju==3.2.0.1
     pytest-mock
     pytest-operator
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/sharding_tests/test_sharding.py
@@ -198,12 +198,12 @@ description = Run all integration tests
 pass_env =
     {[testenv]pass_env}
     CI
-    CI_PACKED_CHARMS
 deps =
     pytest
     juju==3.2.0.1
     pytest-mock
     pytest-operator
+    git+https://github.com/canonical/[email protected]\#subdirectory=python/pytest_plugins/pytest_operator_cache
     -r {tox_root}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s --durations=0 {posargs} {[vars]tests_path}/integration/