cluster: Addition of new tests to run Scrub and Recovery

Add 2 new tests similar to the background Recovery test for running Scrub load with client IO and the other one for running Scrub, Recovery, and Client load.
The Scrub and client IO test performs the following steps:
	- Create a pool and image to populate scrub objects (scrub pool)
	- Create scrub thread
	- Populate the scrub pool with objects using radosbench
	- Initiate deep-scrub on the scrub pool
	- Create a second pool and an image in it to run client IO
	- Initiate fio job on the second image at the same time the deep-scrub starts

In the second test, we have an additional recovery pool that is populated after an OSD is marked down and out.  Once the pool is populated we mark the OSD up and in which starts backfill. At the same time, we begin deep-scrub on the scrub pool and client IO.

Signed-off-by: Aishwarya Mathuria <[email protected]>

benchmark/fio.py

@@ -68,6 +68,9 @@ def initialize_endpoints(self):
             self.client_endpoints_object.create_recovery_image()
         if 'scrubbing_test' in self.cluster.config:
             self.client_endpoints_object.create_scrubbing_image()
+        if 'scrub_recov_test' in self.cluster.config:
+            self.client_endpoints_object.create_recovery_image()
+            self.client_endpoints_object.create_scrubbing_image()
         self.create_endpoints()
 
     def create_endpoints(self):
@@ -221,6 +224,12 @@ def run(self):
             self.cluster.create_scrubbing_test(self.run_dir, scrubbing_callback)
             self.cluster.wait_start_io()
 
+        if 'scrub_recov_test' in self.cluster.config:
+            logger.info('Scrub+Recov')
+            scrub_recov_callback = self.scrub_recov_callback
+            self.cluster.create_scrub_recovery_test(self.run_dir, scrub_recov_callback)
+            self.cluster.wait_start_io()
+
         monitoring.start(self.run_dir)
 
         logger.info('Running fio %s test.', self.mode)
@@ -237,6 +246,10 @@ def run(self):
         if 'scrubbing_test' in self.cluster.config:
             self.cluster.wait_scrubbing_done()
 
+        if 'scrub_recov_test' in self.cluster.config:
+            self.cluster.wait_scrub_recovery_done()
+
+
         monitoring.stop(self.run_dir)
 
         # Finally, get the historic ops
@@ -253,6 +266,9 @@ def recovery_callback_background(self):
     def scrubbing_callback(self):
         logger.info('Scrubbing thread completed')
 
+    def scrub_recov_callback(self):
+        logger.info('Scrub+Recovery thread completed')
+
     def analyze(self, out_dir):
         logger.info('Convert results to json format.')
         for client in settings.getnodes('clients').split(','):

benchmark/librbdfio.py

@@ -88,6 +88,11 @@ def initialize(self):
         # Create the recovery image based on test type requested
         if 'recovery_test' in self.cluster.config and self.recov_test_type == 'background':
             self.mkrecovimage()
+
+        if 'scrub_recov_test' in self.cluster.config:
+            self.mkrecovimage()
+            self.mkscrubimage()
+
         self.mkimages()
         # populate the fio files
         ps = []
@@ -135,9 +140,15 @@ def run(self):
 
         if 'scrubbing_test' in self.cluster.config:
             scrubbing_callback = self.scrubbing_callback
-            self.cluster.create_scrub_test(self.run_dir, scrubbing_callback)
+            self.cluster.create_scrubbing_test(self.run_dir, scrubbing_callback)
             self.cluster.wait_start_io()
 
+        if 'scrub_recov_test' in self.cluster.config:
+            scrub_recov_callback = self.scrub_recov_callback
+            self.cluster.create_scrub_recovery_test(self.run_dir, scrub_recov_callback)
+            self.cluster.wait_start_io()
+
+
         monitoring.start(self.run_dir)
 
         logger.info('Running rbd fio %s test.', self.mode)
@@ -152,6 +163,9 @@ def run(self):
         if 'recovery_test' in self.cluster.config:
             self.cluster.wait_recovery_done()
 
+        if 'scrub_recov_test' in self.cluster.config:
+            self.cluster.wait_scrub_recovery_done()
+
         monitoring.stop(self.run_dir)
 
         # Finally, get the historic ops

client_endpoints/ceph_client_endpoints.py

@@ -123,7 +123,7 @@ def create_rbd_scrubbing(self):
         for node in common.get_fqdn_list('clients'):
             for ep_num in range(0, self.endpoints_per_client):
                 rbd_name = '%s-%s' % (self.pool, self.get_rbd_name(node, ep_num))
-                self.cluster.mkimage(rbd_name, self.get_endpoint_size, self.pool, self.data_pool, self.order)
+                self.cluster.mkimage(rbd_name, self.endpoint_size, self.pool, self.data_pool, self.order)
 
     def mount_rbd(self):
         for ep_num in range(0, self.endpoints_per_client):

client_endpoints/cephfsfuse_client_endpoints.py

@@ -15,6 +15,3 @@ def mount_fs_helper(self, node, dir_name):
 
     def create_recovery_image(self):
         self.create_rbd_recovery()
-
-    def create_scrubbing_image(self):
-        self.create_rbd_scrubbing()

client_endpoints/cephfskernel_client_endpoints.py

@@ -15,6 +15,3 @@ def mount_fs_helper(self, node, dir_name):
 
     def create_recovery_image(self):
         self.create_rbd_recovery()
-
-    def create_scrubbing_image(self):
-        self.create_rbd_scrubbing()