Add force-merge operations to nyc_taxis challenges (elastic#127)

To help comparing the effect of force-merge down to 1 segment, add force-merge/refresh-after-force-merge/ wait-until-merges-finish operations in a number of nyc_taxis challenges. Also add optional max-num-segments to force-merge down to.
cmanning09 · Jun 17, 2020 · 27cd4be · 27cd4be
1 parent 835c4ce
commit 27cd4be
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 25 deletions.
diff --git a/nyc_taxis/README.md b/nyc_taxis/README.md
@@ -55,7 +55,7 @@ bzip2 -k documents.json
 
 ### Parameters
 
-This track allows to overwrite the following parameters with Rally 0.8.0+ using `--track-params`:
+This track allows to overwrite the following parameters using `--track-params`:
 
 * `bulk_size` (default: 10000)
 * `bulk_indexing_clients` (default: 8): Number of clients that issue bulk indexing requests.
@@ -67,6 +67,7 @@ This track allows to overwrite the following parameters with Rally 0.8.0+ using
 * `number_of_replicas` (default: 0)
 * `number_of_shards` (default: 1)
 * `source_enabled` (default: true): A boolean defining whether the `_source` field is stored in the index.
+* `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use.
 * `index_settings`: A list of index settings. Index settings defined elsewhere (e.g. `number_of_replicas`) need to be overridden explicitly.
 * `cluster_health` (default: "green"): The minimum required cluster health.
 

diff --git a/nyc_taxis/challenges/default.json b/nyc_taxis/challenges/default.json
@@ -36,6 +36,21 @@
           "name": "refresh-after-index",
           "operation": "refresh"
         },
+        {
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200{%- if force_merge_max_num_segments is defined %},
+            "max-num-segments": {{ force_merge_max_num_segments | tojson }}
+            {%- endif %}
+          }
+        },
+        {
+          "name": "refresh-after-force-merge",
+          "operation": "refresh"
+        },
+        {
+          "operation": "wait-until-merges-finish"
+        },
         {
           "operation": "default",
           "warmup-iterations": 50,
@@ -104,6 +119,21 @@
         {
           "name": "refresh-after-index",
           "operation": "refresh"
+        },
+        {
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200{%- if force_merge_max_num_segments is defined %},
+            "max-num-segments": {{ force_merge_max_num_segments | tojson }}
+            {%- endif %}
+          }
+        },
+        {
+          "name": "refresh-after-force-merge",
+          "operation": "refresh"
+        },
+        {
+          "operation": "wait-until-merges-finish"
         }
       ]
     },
@@ -145,6 +175,21 @@
         {
           "name": "refresh-after-index",
           "operation": "refresh"
+        },
+        {
+          "operation": {
+            "operation-type": "force-merge",
+            "request-timeout": 7200{%- if force_merge_max_num_segments is defined %},
+            "max-num-segments": {{ force_merge_max_num_segments | tojson }}
+            {%- endif %}
+          }
+        },
+        {
+          "name": "refresh-after-force-merge",
+          "operation": "refresh"
+        },
+        {
+          "operation": "wait-until-merges-finish"
         }
       ]
     },
@@ -187,25 +232,17 @@
         {
           "operation": {
             "operation-type": "force-merge",
-            "request-timeout": 7200
+            "request-timeout": 7200{%- if force_merge_max_num_segments is defined %},
+            "max-num-segments": {{ force_merge_max_num_segments | tojson }}
+            {%- endif %}
           }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh"
         },
         {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
+          "operation": "wait-until-merges-finish"
         }
       ]
     },
@@ -418,25 +455,17 @@
         {
           "operation": {
             "operation-type": "force-merge",
-            "request-timeout": 7200
+            "request-timeout": 7200{%- if force_merge_max_num_segments is defined %},
+            "max-num-segments": {{ force_merge_max_num_segments | tojson }}
+            {%- endif %}
           }
         },
         {
           "name": "refresh-after-force-merge",
           "operation": "refresh"
         },
         {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
+          "operation": "wait-until-merges-finish"
         },
         {
           "operation": {

diff --git a/nyc_taxis/operations/default.json b/nyc_taxis/operations/default.json
@@ -14,6 +14,17 @@
       "conflict-probability": {{conflict_probability | default(25)}},
       "recency": {{recency | default(0)}}
     },
+    {
+      "name": "wait-until-merges-finish",
+      "operation-type": "index-stats",
+      "index": "_all",
+      "condition": {
+        "path": "_all.total.merges.current",
+        "expected-value": 0
+      },
+      "retry-until-success": true,
+      "include-in-reporting": false
+    },
     {
       "name": "default",
       "operation-type": "search",