Merge pull request #1201 from lmlg/osd-key-rotate

Implement key rotation tests for OSD units

zaza/openstack/charm_tests/ceph/tests.py

@@ -1795,9 +1795,25 @@ def entity_filter(name):
             action_params={'entity': entity}
         )
         zaza_utils.assertActionRanOK(action_obj)
-        zaza_model.wait_for_application_states(states=self.app_states)
-        new_keys = self._get_all_keys(unit, entity_filter)
-        self.assertNotEqual(old_keys, new_keys)
+        # NOTE(lmlg): There's a nasty race going on here. Essentially,
+        # since this action involves 2 different applications, what
+        # happens is as follows:
+        #          (1)            (2)               (3)              (4)
+        # ceph-mon rotates key | (idle) | remote-unit rotates key | (idle)
+        # Between (2) and (3), there's a window where all units are
+        # idle, _but_ the key hasn't been rotated in the other unit.
+        # As such, we retry a few times instead of using the
+        # `wait_for_application_states` interface.
+
+        for attempt in tenacity.Retrying(
+            wait=tenacity.wait_exponential(multiplier=2, max=32),
+            reraise=True, stop=tenacity.stop_after_attempt(20),
+            retry=tenacity.retry_if_exception_type(AssertionError)
+        ):
+            with attempt:
+                new_keys = self._get_all_keys(unit, entity_filter)
+                self.assertNotEqual(old_keys, new_keys)
+
         diff = new_keys - old_keys
         self.assertEqual(len(diff), 1)
         first = next(iter(diff))
@@ -1822,7 +1838,7 @@ def _get_fs_client(self, unit):
     def test_key_rotate(self):
         """Test that rotating the keys actually changes them."""
         unit = 'ceph-mon/0'
-        self._check_key_rotation('mgr', unit)
+        self._check_key_rotation('osd.0', unit)
 
         try:
             zaza_model.get_application('ceph-radosgw')