cluster: Addition of new tests to run Scrub and Recovery

Add 2 new tests similar to the background Recovery test for running Scrub load with client IO and the other one for running Scrub, Recovery, and Client load.
The Scrub and client IO test performs the following steps:
	- Create a pool and image to populate scrub objects (scrub pool)
	- Create scrub thread
	- Populate the scrub pool with objects using radosbench
	- Initiate deep-scrub on the scrub pool
	- Create a second pool and an image in it to run client IO
	- Initiate fio job on the second image at the same time the deep-scrub starts

In the second test, we have an additional recovery pool that is populated after an OSD is marked down and out.  Once the pool is populated we mark the OSD up and in which starts backfill. At the same time, we begin deep-scrub on the scrub pool and client IO.

Signed-off-by: Aishwarya Mathuria <[email protected]>

benchmark/fio.py

@@ -66,6 +66,11 @@ def initialize_endpoints(self):
         # Create the recovery image based on test type requested
         if 'recovery_test' in self.cluster.config and self.recov_test_type == 'background':
             self.client_endpoints_object.create_recovery_image()
+        if 'scrubbing_test' in self.cluster.config:
+            self.client_endpoints_object.create_scrubbing_image()
+        if 'scrub_recov_test' in self.cluster.config:
+            self.client_endpoints_object.create_recovery_image()
+            self.client_endpoints_object.create_scrubbing_image()
         self.create_endpoints()
 
     def create_endpoints(self):
@@ -213,6 +218,18 @@ def run(self):
             # Wait for signal to start client IO
             self.cluster.wait_start_io()
 
+        if 'scrubbing_test' in self.cluster.config:
+            logger.info('Scrubbing test in config')
+            scrubbing_callback = self.scrubbing_callback
+            self.cluster.create_scrubbing_test(self.run_dir, scrubbing_callback)
+            self.cluster.wait_start_io()
+
+        if 'scrub_recov_test' in self.cluster.config:
+            logger.info('Scrub+Recov')
+            scrub_recov_callback = self.scrub_recov_callback
+            self.cluster.create_scrub_recovery_test(self.run_dir, scrub_recov_callback)
+            self.cluster.wait_start_io()
+
         monitoring.start(self.run_dir)
 
         logger.info('Running fio %s test.', self.mode)
@@ -225,6 +242,13 @@ def run(self):
         # If we were doing recovery, wait until it's done.
         if 'recovery_test' in self.cluster.config:
             self.cluster.wait_recovery_done()
+        # If we were doing scrubbing, wait until it's done.
+        if 'scrubbing_test' in self.cluster.config:
+            self.cluster.wait_scrubbing_done()
+
+        if 'scrub_recov_test' in self.cluster.config:
+            self.cluster.wait_scrub_recovery_done()
+
 
         monitoring.stop(self.run_dir)
 
@@ -239,6 +263,12 @@ def recovery_callback_blocking(self):
     def recovery_callback_background(self):
         logger.info('Recovery thread completed!')
 
+    def scrubbing_callback(self):
+        logger.info('Scrubbing thread completed')
+
+    def scrub_recov_callback(self):
+        logger.info('Scrub+Recovery thread completed')
+
     def analyze(self, out_dir):
         logger.info('Convert results to json format.')
         for client in settings.getnodes('clients').split(','):

benchmark/librbdfio.py

@@ -46,6 +46,8 @@ def __init__(self, archive_dir, cluster, config):
         self.use_existing_volumes = config.get('use_existing_volumes', False)
         self.pool_name = config.get("poolname", "cbt-librbdfio")
         self.recov_pool_name = config.get("recov_pool_name", "cbt-librbdfio-recov")
+        self.scrub_pool_name = config.get("scrub_pool_name", "cbt-librbdfio-scrub")
+        self.scrub_pool_profile = config.get("scrub_pool_profile", "default")
         self.rbdname = config.get('rbdname', '')
 
         self.total_procs = self.procs_per_volume * self.volumes_per_client * len(settings.getnodes('clients').split(','))
@@ -80,9 +82,17 @@ def initialize(self):
 
         common.sync_files('%s/*' % self.run_dir, self.out_dir)
 
+        if 'scrubbing_test' in self.cluster.config:
+            self.mkscrubimage()
+
         # Create the recovery image based on test type requested
         if 'recovery_test' in self.cluster.config and self.recov_test_type == 'background':
             self.mkrecovimage()
+
+        if 'scrub_recov_test' in self.cluster.config:
+            self.mkrecovimage()
+            self.mkscrubimage()
+
         self.mkimages()
         # populate the fio files
         ps = []
@@ -128,6 +138,17 @@ def run(self):
             # Wait for a signal from the recovery thread to initiate client IO
             self.cluster.wait_start_io()
 
+        if 'scrubbing_test' in self.cluster.config:
+            scrubbing_callback = self.scrubbing_callback
+            self.cluster.create_scrubbing_test(self.run_dir, scrubbing_callback)
+            self.cluster.wait_start_io()
+
+        if 'scrub_recov_test' in self.cluster.config:
+            scrub_recov_callback = self.scrub_recov_callback
+            self.cluster.create_scrub_recovery_test(self.run_dir, scrub_recov_callback)
+            self.cluster.wait_start_io()
+
+
         monitoring.start(self.run_dir)
 
         logger.info('Running rbd fio %s test.', self.mode)
@@ -142,6 +163,9 @@ def run(self):
         if 'recovery_test' in self.cluster.config:
             self.cluster.wait_recovery_done()
 
+        if 'scrub_recov_test' in self.cluster.config:
+            self.cluster.wait_scrub_recovery_done()
+
         monitoring.stop(self.run_dir)
 
         # Finally, get the historic ops
@@ -210,6 +234,18 @@ def mkrecovimage(self):
                   self.cluster.mkimage('cbt-librbdfio-recov-%s-%d' % (node,volnum), self.vol_size, self.recov_pool_name, self.data_pool, self.vol_object_size)
         monitoring.stop()
 
+    def mkscrubimage(self):
+        logger.info('Creating scrubbing image...')
+        monitoring.start("%s/scrub_pool_monitoring" % self.run_dir)
+        if (self.use_existing_volumes == False):
+            self.cluster.rmpool(self.scrub_pool_name, self.scrub_pool_profile)
+            self.cluster.mkpool(self.scrub_pool_name, self.scrub_pool_profile, 'rbd')
+            for node in common.get_fqdn_list('clients'):
+                for volnum in range(0, self.volumes_per_client):
+                    node = node.rpartition("@")[2]
+                    self.cluster.mkimage('cbt-librbdfio-scrub-%s-%d' % (node,volnum), self.vol_size, self.scrub_pool_name, self.data_pool, self.vol_object_size)
+        monitoring.stop()
+
     def mkimages(self):
         monitoring.start("%s/pool_monitoring" % self.run_dir)
         if (self.use_existing_volumes == False):
@@ -231,6 +267,9 @@ def recovery_callback_blocking(self):
     def recovery_callback_background(self):
         logger.info('Recovery thread completed!')
 
+    def scrubbing_callback(self):
+        logger.info('Scrubbing thread completed!')
+
     def parse(self, out_dir):
         for client in settings.getnodes('clients').split(','):
             host = settings.host_info(client)["host"]

client_endpoints/ceph_client_endpoints.py

@@ -26,6 +26,8 @@ def __init__(self, cluster, config):
         self.data_pool_profile = config.get('data_pool_profile', None)
         self.recov_pool = None
         self.recov_pool_profile = config.get('recov_pool_profile', 'default')
+        self.scrub_pool = None
+        self.scrub_pool_profile = config.get('scrub_pool_profile', 'default')
         self.order = config.get('order', 22)
         self.disabled_features = config.get('disabled_features', None)
 
@@ -114,6 +116,15 @@ def create_rbd_recovery(self):
                 rbd_name = '%s-%s' % (self.pool, self.get_rbd_name(node, ep_num))
                 self.cluster.mkimage(rbd_name, self.endpoint_size, self.pool, self.data_pool, self.order)
 
+    def create_rbd_scrubbing(self):
+        self.pool = '%s-scrub' % self.name
+        self.cluster.rmpool(self.pool, self.scrub_pool_profile)
+        self.cluster.mkpool(self.pool, self.scrub_pool_profile, 'rbd')
+        for node in common.get_fqdn_list('clients'):
+            for ep_num in range(0, self.endpoints_per_client):
+                rbd_name = '%s-%s' % (self.pool, self.get_rbd_name(node, ep_num))
+                self.cluster.mkimage(rbd_name, self.endpoint_size, self.pool, self.data_pool, self.order)
+
     def mount_rbd(self):
         for ep_num in range(0, self.endpoints_per_client):
             dir_name = self.get_dir_name(ep_num)

client_endpoints/client_endpoints.py

@@ -45,3 +45,6 @@ def remove(self):
 
     def create_recovery_image(self):
         pass
+
+    def create_scrubbing_image(self):
+        pass

client_endpoints/librbd_client_endpoints.py

@@ -19,3 +19,6 @@ def mount(self):
 
     def create_recovery_image(self):
         self.create_rbd_recovery()
+
+    def create_scrubbing_image(self):
+        self.create_rbd_scrubbing()

client_endpoints/rbdfuse_client_endpoints.py

@@ -31,3 +31,6 @@ def map_rbd(self, node, rbd_name):
 
     def create_recovery_image(self):
         self.create_rbd_recovery()
+
+    def create_scrubbing_image(self):
+        self.create_rbd_scrubbing()

client_endpoints/rbdkernel_client_endpoints.py

@@ -22,3 +22,6 @@ def map_rbd(self, node, rbd_name):
 
     def create_recovery_image(self):
         self.create_rbd_recovery()
+
+    def create_scrubbing_image(self):
+        self.create_rbd_scrubbing()

client_endpoints/rbdnbd_client_endpoints.py

@@ -16,3 +16,6 @@ def map_rbd(self, node, rbd_name):
 
     def create_recovery_image(self):
         self.create_rbd_recovery()
+
+    def create_scrubbing_image(self):
+        self.create_rbd_scrubbing()

client_endpoints/rbdtcmu_client_endpoints.py

@@ -23,3 +23,6 @@ def map_rbd(self, node, rbd_name):
 
     def create_recovery_image(self):
         self.create_rbd_recovery()
+
+    def create_scrubbing_image(self):
+        self.create_rbd_scrubbing()