Add changes to benchmarks to securely pass Cluster endpoint details (#…

…4621)

Signed-off-by: Divya Madala <[email protected]>

jenkins/opensearch/benchmark-test-endpoint.jenkinsfile

@@ -7,7 +7,7 @@
  * compatible open source license.
  */
 
-lib = library(identifier: '[email protected].1', retriever: modernSCM([
+lib = library(identifier: '[email protected].3', retriever: modernSCM([
 
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
@@ -21,20 +21,29 @@ pipeline {
     }
     environment {
         AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test'
+        JOB_NAME = 'benchmark-test'
     }
 parameters {
-        string(
+        password(
             name: 'CLUSTER_ENDPOINT',
             description: 'Provide an endpoint to a cluster for running benchmark tests against it.',
-            trim: true
         )
         booleanParam(
             name: 'SECURITY_ENABLED',
             description: 'Mention if the cluster is secured or insecured.',
             defaultValue: false,
         )
+        password(
+            name: 'USERNAME',
+            description: 'Enter username for the cluster endpoint',
+            defaultValue: ''
+        )
+        password(
+            name: 'PASSWORD',
+            description: 'Enter password for the cluster endpoint',
+            defaultValue: ''
+        )
         string(
-
             name: 'TEST_WORKLOAD',
             description: 'The workload name from OpenSearch Benchmark Workloads.',
             defaultValue: 'nyc_taxis',
@@ -108,6 +117,8 @@ parameters {
                     runBenchmarkTestScript(
                         endpoint: CLUSTER_ENDPOINT,
                         insecure: !(params.SECURITY_ENABLED),
+                        username: USERNAME,
+                        password: PASSWORD,
                         workload: TEST_WORKLOAD,
                         userTag: USER_TAGS.isEmpty() ? "security-enabled:${SECURITY_ENABLED}" : "${USER_TAGS},security-enabled:${SECURITY_ENABLED}",
                         workloadParams: WORKLOAD_PARAMS,

run.sh

@@ -20,5 +20,5 @@ echo "Installing dependencies in $DIR ..."
 export PIPENV_PIPFILE="$DIR/Pipfile"
 python3 -m pipenv install
 
-echo "Running "$1" ${@:2} ..."
+if [[ ! "$1" == *benchmark* ]]; then (echo "Running "$1" ${@:2} ..."; ); fi
 python3 -m pipenv run python "$1" ${@:2}

src/test_workflow/benchmark_test/benchmark_args.py

@@ -59,6 +59,8 @@ def __init__(self) -> None:
                             help="Load balancer url for benchmark testing")
         parser.add_argument("--distribution-version", dest="distribution_version",
                             help="provide OpenSearch version if using distribution-url param.")
+        parser.add_argument("--username", dest="username", help="Username for the cluster")
+        parser.add_argument("--password", dest="password", help="Password for the cluster")
         parser.add_argument("--suffix", dest="suffix", help="Suffix to be added to stack name for performance test")
         parser.add_argument("--component", dest="component", default="OpenSearch",
                             help="Component name that needs to be performance tested")
@@ -133,6 +135,8 @@ def __init__(self) -> None:
         self.min_distribution = args.min_distribution
         self.component = args.component
         self.insecure = args.insecure
+        self.username = args.username if args.username else "admin"
+        self.password = args.password if args.password else None
         self.manager_node_count = args.manager_node_count if args.manager_node_count else None
         self.data_node_count = args.data_node_count if args.data_node_count else None
         self.client_node_count = args.client_node_count if args.client_node_count else None

src/test_workflow/benchmark_test/benchmark_create_cluster.py

@@ -64,6 +64,7 @@ def __init__(
             f" -c assume-role-credentials:writeIamRoleName={role} -c assume-role-credentials:readIamRoleName={role} "
         )
         self.params = "".join(params_list) + role_params
+        self.password = None if self.args.insecure else get_password(self.args.distribution_version)
         self.is_endpoint_public = False
         self.stack_name = f"opensearch-infra-stack-{self.args.stack_suffix}"
         if self.manifest:

src/test_workflow/benchmark_test/benchmark_test_cluster.py

@@ -32,22 +32,23 @@ def __init__(
         self.args = args
         self.cluster_endpoint = self.args.cluster_endpoint
         self.cluster_endpoint_with_port = None
-        self.password = None
+        self.password = self.args.password if self.args.password else get_password('2.12.0')
 
     def start(self) -> None:
 
-        command = f"curl http://{self.cluster_endpoint}" if self.args.insecure else f"curl https://{self.cluster_endpoint} -ku 'admin:{get_password('2.12.0')}'"
+        command = f"curl http://{self.cluster_endpoint}" if self.args.insecure else f"curl https://{self.cluster_endpoint} -ku '{self.args.username}:{self.password}'"
         try:
-            result = subprocess.run(command, shell=True, capture_output=True, timeout=5)
+            result = subprocess.run(command, shell=True, capture_output=True, timeout=30)
         except subprocess.TimeoutExpired:
-            raise TimeoutError(f"Time out! Couldn't connect to the cluster {self.cluster_endpoint}")
+            raise TimeoutError("Time out! Couldn't connect to the cluster")
 
         if result.stdout:
             res_dict = json.loads(result.stdout)
             self.args.distribution_version = res_dict['version']['number']
-        self.wait_for_processing()
-
-        self.cluster_endpoint_with_port = "".join([self.cluster_endpoint, ":", str(self.port)])
+            self.wait_for_processing()
+            self.cluster_endpoint_with_port = "".join([self.cluster_endpoint, ":", str(self.port)])
+        else:
+            raise Exception("Empty response retrieved from the curl command")
 
     @property
     def endpoint(self) -> str:
@@ -65,10 +66,9 @@ def fetch_password(self) -> str:
         return self.password
 
     def wait_for_processing(self, tries: int = 3, delay: int = 15, backoff: int = 2) -> None:
-        logging.info(f"Waiting for domain at {self.endpoint} to be up")
+        logging.info("Waiting for domain ******* to be up")
         protocol = "http://" if self.args.insecure else "https://"
         url = "".join([protocol, self.endpoint, "/_cluster/health"])
-        self.password = None if self.args.insecure else get_password(self.args.distribution_version)
-        request_args = {"url": url} if self.args.insecure else {"url": url, "auth": HTTPBasicAuth("admin", self.password),  # type: ignore
+        request_args = {"url": url} if self.args.insecure else {"url": url, "auth": HTTPBasicAuth(self.args.username, self.password),  # type: ignore
                                                                 "verify": False}  # type: ignore
         retry_call(requests.get, fkwargs=request_args, tries=tries, delay=delay, backoff=backoff)

src/test_workflow/benchmark_test/benchmark_test_suite.py

@@ -69,8 +69,9 @@ def __init__(
 
     def execute(self) -> None:
         if self.security:
-            self.command += f' --client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'{self.password}\'"'
+            self.command += f' --client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'{self.args.username}\',basic_auth_password:\'{self.password}\'"'
         else:
             self.command += ' --client-options="timeout:300"'
-        logging.info(f"Executing {self.command}")
+        log_info = f"Executing {self.command.replace(self.endpoint, len(self.endpoint) * '*').replace(self.args.username, len(self.args.username) * '*')}"
+        logging.info(log_info.replace(self.password, len(self.password) * '*') if self.password else log_info)
         subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True)

test.sh

@@ -21,7 +21,6 @@ case $1 in
   "$DIR/run.sh" "$DIR/src/run_perf_test.py" "${@:2}"
   ;;
   "benchmark-test")
-  echo "the parameters passed are ${@:2}"
   "$DIR/run.sh" "$DIR/src/run_benchmark_test.py" "${@:2}"
   ;;
   *)

tests/jenkins/TestRunBenchmarkTestEndpoint.groovy

@@ -26,7 +26,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{
     void setUp() {
         helper.registerSharedLibrary(
                 library().name('jenkins')
-                        .defaultVersion('6.4.1')
+                        .defaultVersion('6.4.3')
                         .allowOverride(true)
                         .implicit(true)
                         .targetPath('vars')
@@ -60,6 +60,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{
         binding.setVariable('EXCLUDE_TASKS', '')
         binding.setVariable('INCLUDE_TASKS', '')
         binding.setVariable('ADDITIONAL_CONFIG', '')
+        binding.setVariable('JOB_NAME', 'benchmark-test')
         binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config')
         binding.setVariable('STAGE_NAME', 'test_stage')
         binding.setVariable('TEST_WORKLOAD', 'nyc-taxis')
@@ -105,7 +106,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{
 
         assertThat(testScriptCommands.size(), equalTo(1))
         assertThat(testScriptCommands, hasItems(
-                "./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                --test-procedure append-no-conflicts       --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
+                "set +x && ./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                  --test-procedure append-no-conflicts       --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
         ))
     }
     @Test
@@ -126,7 +127,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{
 
         assertThat(testScriptCommands.size(), equalTo(1))
         assertThat(testScriptCommands, hasItems(
-                "./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security               --test-procedure append-no-conflicts       --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
+                "set +x && ./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security                 --test-procedure append-no-conflicts       --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
         ))
     }
 

tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt

@@ -1,6 +1,6 @@
    benchmark-test-endpoint.run()
       benchmark-test-endpoint.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
-      benchmark-test-endpoint.library({[email protected].1, retriever=null})
+      benchmark-test-endpoint.library({[email protected].3, retriever=null})
       benchmark-test-endpoint.pipeline(groovy.lang.Closure)
          benchmark-test-endpoint.timeout({time=24, unit=HOURS})
          benchmark-test-endpoint.logRotator({daysToKeepStr=30})
@@ -15,9 +15,9 @@
             benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host])
             benchmark-test-endpoint.script(groovy.lang.Closure)
                benchmark-test-endpoint.echo(security-enabled: true)
-               benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
+               benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, username=, password=, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
                   runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
-                  runBenchmarkTestScript.library({[email protected].1, retriever=null})
+                  runBenchmarkTestScript.library({[email protected].3, retriever=null})
                   runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
                   runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
                   runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
@@ -28,6 +28,6 @@
                   runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure)
                      runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini})
                      runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=})
-                  runBenchmarkTestScript.sh(./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                --test-procedure append-no-conflicts       --telemetry-params '{"telemetry_setting":"value"}')
+                  runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                  --test-procedure append-no-conflicts       --telemetry-params '{"telemetry_setting":"value"}')
          benchmark-test-endpoint.postCleanup()
             postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})

tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt

@@ -1,6 +1,6 @@
    benchmark-test-endpoint.run()
       benchmark-test-endpoint.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
-      benchmark-test-endpoint.library({[email protected].1, retriever=null})
+      benchmark-test-endpoint.library({[email protected].3, retriever=null})
       benchmark-test-endpoint.pipeline(groovy.lang.Closure)
          benchmark-test-endpoint.timeout({time=24, unit=HOURS})
          benchmark-test-endpoint.logRotator({daysToKeepStr=30})
@@ -15,9 +15,9 @@
             benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host])
             benchmark-test-endpoint.script(groovy.lang.Closure)
                benchmark-test-endpoint.echo(security-enabled: true)
-               benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
+               benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, username=, password=, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
                   runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
-                  runBenchmarkTestScript.library({[email protected].1, retriever=null})
+                  runBenchmarkTestScript.library({[email protected].3, retriever=null})
                   runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
                   runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
                   runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
@@ -28,6 +28,6 @@
                   runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure)
                      runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini})
                      runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=})
-                  runBenchmarkTestScript.sh(./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                --test-procedure append-no-conflicts       --telemetry-params '{"telemetry_setting":"value"}')
+                  runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                  --test-procedure append-no-conflicts       --telemetry-params '{"telemetry_setting":"value"}')
          benchmark-test-endpoint.postCleanup()
             postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})