diff --git a/.gitignore b/.gitignore index e3569bd..2928a7c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ results istioctl* +sampled_stats/* \ No newline at end of file diff --git a/DeploymentExample.yml b/DeploymentExample.yml new file mode 100644 index 0000000..4de1dc3 --- /dev/null +++ b/DeploymentExample.yml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: scale-test + version: "1.0" + name: app-2-n + namespace: mesh-scale +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: scale-test + app.index: application-1 + app.variant: stable + deploymentconfig: app-2 + version: "1.0" + strategy: + activeDeadlineSeconds: 21600 + recreateParams: + timeoutSeconds: 600 + resources: {} + type: Recreate + template: + metadata: + annotations: + sidecar.istio.io/concurrency: "1" + sidecar.istio.io/inject: "true" + sidecar.istio.io/proxyCPU: 200m + sidecar.istio.io/proxyCPULimit: 1000m + sidecar.istio.io/statsInclusionPrefixes: listener,cluster.outbound + creationTimestamp: null + labels: + app: scale-test + app.index: application-1 + app.variant: stable + deploymentconfig: app-2 + version: "1.0" + spec: + containers: + - env: + - name: NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: VARIANT + value: canary + - name: JAVA_OPTIONS + value: | + -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.Log4j2LogDelegateFactory + image: quay.io/cfilleke/mannequin:0.0-z + imagePullPolicy: Always + name: mannequin + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + exec: + command: + - curl + - localhost:8080/ + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 300m + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/certs + name: certs + dnsPolicy: ClusterFirst + nodeSelector: + test.role: workload + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + tolerations: + - effect: NoExecute + key: workload + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: scale-test + maxSkew: 1 + topologyKey: workload + whenUnsatisfiable: ScheduleAnyway + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: app-1 + test: false + triggers: + - type: ConfigChange \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a9a24b9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +#### +# This Dockerfile is used in order to build a container that runs a +# Quarkus application in JVM mode on s390x architecture (IBM System Z) +# +# Before building the docker image run: +# +# mvn package +# +# Then, build the image with: +# +# podman build -f Dockerfile.Z -t quay.io//mannequin:0.0-z . +# +# Then run the container using: +# +# podman run -i --rm -p 8080:8080 quarkus/mannequin-jvm quay.io//mannequin:0.0-z . +# +# the instructions here are to put it in a public registry (e.g. quay.io) +# so that you can pull it into a pod deployment on OpenShift +# +### +FROM registry.access.redhat.com/ubi9/ubi:latest +RUN yum install -y java-11-openjdk.s390x +ENV JAVA_OPTIONS=-Dquarkus.http.host=0.0.0.0 +COPY target/lib/* /deployments/lib/ +COPY target/*-runner.jar /deployments/app.jar +COPY ./java-runner.sh /deployments/java-runner.sh +RUN chmod a+x /deployments/java-runner.sh +ENTRYPOINT [ "/deployments/java-runner.sh" ] diff --git a/README.md b/README.md index ab7580b..e7ed92c 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,39 @@ # Istio/Maistra scalability tests -To get this benchmark running: +## Prerequisites -1. Install OCP and Ansible -2. Login to OCP: `oc login -u system:admin` -3. Install Istio: https://maistra.io/docs/getting_started/install/ - - In `controlplane/basic-install` set `gateways.ior_enabled: true` and `mixer.telemetry.enabled: false` - - I suggest locating `istio-system` pods on the infra node (the same where the `default/router` resides): - `oc patch namespace istio-system -p '{"metadata":{"annotations":{"openshift.io/node-selector":"node-role.kubernetes.io/infra=true"}}}'` - - I had trouble with `clusterrole istio-sidecar-injector-istio-system` - this was not correctly created and I had to fix it manually, applying: -``` - apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-sidecar-injector-istio-system - labels: - app: istio-sidecar-injector - release: istio - istio: sidecar-injector -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch"] -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations"] - verbs: ["get", "list", "watch", "patch", "create" ] -``` -4. You might need to add the policies: - ``` - oc adm policy add-scc-to-user anyuid -z istio-ingress-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z default -n istio-system - oc adm policy add-scc-to-user anyuid -z prometheus -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-egressgateway-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-citadel-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-ingressgateway-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-cleanup-old-ca-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-mixer-post-install-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-mixer-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-pilot-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-sidecar-injector-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-galley-service-account -n istio-system - oc adm policy add-scc-to-user anyuid -z istio-security-post-install-account -n istio-system - ``` -5. Allow wildcard routes: `oc set env dc/router ROUTER_ALLOW_WILDCARD_ROUTES=true -n default` (not possible in OCP 4.1) -6. Create hosts.* according to your system -7. Run the setup (now everything should be automatized): - `ansible-playbook -i hosts.mysetup setup.yaml` -8. Start the test: - `ansible-playbook -i hosts.mysetup test.yaml` +### 1. OCP +### 2. Ansible +### 3. Hyperfoil + +https://github.ibm.com/Openshift-Addons-Performance/Openshift-performance-Docs-IBM-Z + +### 4. Firewall + +## Setup + +1. Install prerequisites +2. Run `prep_nodes.sh` to label the nodes. +3. Login to OCP: `oc login -u system:admin` + +## Test with OSSM + 1. Install OSSM + - In `controlplane/basic-install` set `gateways.ior_enabled: true` and `mixer.telemetry.enabled: false` + - I suggest locating `istio-system` pods on the infra node (the same where the `default/router` resides): + `oc patch namespace istio-system -p '{"metadata":{"annotations":{"openshift.io/node-selector":"node-role.kubernetes.io/infra=true"}}}'` + 2. Create hosts.* according to your system + 3. Run the setup (now everything should be automatized): + `ansible-playbook -i hosts.withossm setup.yaml` + 4. Start the test: + `ansible-playbook -i hosts.withossm test.yaml` + +## Test without OSSM + 1. Create hosts.* according to your system + 2. Run the setup (now everything should be automatized): + `ansible-playbook -i hosts.withoutossm setup.yaml` + 3. Start the test: + `ansible-playbook -i hosts.withoutossm test.yaml` + ## Hints: @@ -61,4 +47,4 @@ rules: * There seems to be a bug in IOR (MAISTRA-356) that is not resolved in the image I use. Therefore you have to manually fix the generated route: `oc get route -n istio-system -l maistra.io/generated-by=ior` `oc patch route -n istio-system app-gateway-xxxxx -p '{ "spec": { "port" : { "targetPort": 443 }}}'` TODO -oc get deployment istio-ingressgateway -o json | jq '.spec.template.spec.containers[].resources.requests={},.spec.template.spec.containers[].args += ["--proxy-concurrency", "4"]' \ No newline at end of file +oc get deployment istio-ingressgateway -o json | jq '.spec.template.spec.containers[].resources.requests={},.spec.template.spec.containers[].args += ["--proxy-concurrency", "4"]' diff --git a/benchmarks/closedmodel-withoutossm.yaml.j2 b/benchmarks/closedmodel-withoutossm.yaml.j2 new file mode 100644 index 0000000..e73b5f8 --- /dev/null +++ b/benchmarks/closedmodel-withoutossm.yaml.j2 @@ -0,0 +1,73 @@ +name: closedmodel-withoutossm +agents: +{% if hyperfoil_deployer == "k8s" %} + {{ agent }}: + image: quay.io/ibm-z-performance/hyperfoil:0.25 + pod-cpu: "4000m" + pod-memory: "4Gi" +{% else %} + {{ agent }}: {{ hostvars[agent]['ansible_host'] }}:{{ hyperfoil_agent_port }} +{% endif %} +threads: 50 +http: +{% for index in range(1, (num_apps | int) + 1) %} +- host: https://app-{{ index }}.{{ wildcard_domain }} + allowHttp2: {{ http2 }} + sharedConnections: {{ (test_shared_connections | default(100, true)) // (num_apps | int) }} +{% endfor %} +phases: +- warmup: + always: + isWarmup: true + users: {{ test_concurrent_users }} + duration: {{ test_rampup_duration }} + forks: + simple: &simple + weight: {{ (fork == "simple" or fork == "all") | ternary(1, 0) }} + scenario: + - params: ¶ms + - set: p <- 1 + - randomInt: + toVar: hostIndex + min: 1 + max: {{ num_apps }} + - test: + - httpRequest: + GET: /mersennePrime?p=${p} + authority: app-${hostIndex}.{{ wildcard_domain }} + sla: + errorRatio: 0.1 + proxy: &proxy + weight: {{ (fork == "proxy" or fork == "all") | ternary(1, 0) }} + scenario: + - params: *params + - test: + - httpRequest: + GET: /proxy?p=${p}&url=http://app-${hostIndex}:8080/mersennePrime?p=${p} + authority: app-${hostIndex}.{{ wildcard_domain }} + sla: + errorRatio: 0.1 + db: &db + weight: {{ (fork == "db" or fork == "all") | ternary(1, 0) }} + scenario: + - params: *params + - test: + - randomInt: + toVar: size + min: 5 + max: 20 + - httpRequest: + GET: /db?p=${p}&host=app-${hostIndex}&size=${size} + authority: app-${hostIndex}.{{ wildcard_domain }} + sla: + errorRatio: 0.1 +- steady: + always: + users: {{ test_concurrent_users }} + duration: {{ test_steady_duration }} + startAfter: warmup + forks: + simple: *simple + proxy: *proxy + db: *db + diff --git a/benchmarks/closedmodel.yaml.j2 b/benchmarks/closedmodel.yaml.j2 index bd30a8a..92a27c8 100644 --- a/benchmarks/closedmodel.yaml.j2 +++ b/benchmarks/closedmodel.yaml.j2 @@ -1,17 +1,13 @@ name: closedmodel agents: - default: - extras: -XX:+UseShenandoahGC - # -XX:StartFlightRecording=delay=30s,duration=150s,disk=true,filename=/tmp/recording.jfr,settings=profile - #stop: false -{% for agent in groups[hyperfoil_agent_group] %} {% if hyperfoil_deployer == "k8s" %} {{ agent }}: - node: {{ agent }} + image: quay.io/ibm-z-performance/hyperfoil:0.25 + pod-cpu: "2000m" + pod-memory: "2Gi" {% else %} {{ agent }}: {{ hostvars[agent]['ansible_host'] }}:{{ hyperfoil_agent_port }} {% endif %} -{% endfor %} threads: 50 http: {% for index in range(1, (num_apps | int) + 1) %} @@ -54,7 +50,7 @@ phases: - test: - httpRequest: GET: /mersennePrime?p=${p} - authority: app-${hostIndex}.{{ wildcard_domain }}:443 + authority: app-${hostIndex}.{{ wildcard_domain }} headers: x-variant: fromVar: variant @@ -75,7 +71,7 @@ phases: - test: - httpRequest: GET: /proxy?p=${p}&url=http://app-${hostIndex}:8080/mersennePrime?p=${p} - authority: app-${hostIndex}.{{ wildcard_domain }}:443 + authority: app-${hostIndex}.{{ wildcard_domain }} headers: x-variant: fromVar: variant @@ -103,7 +99,7 @@ phases: max: 20 - httpRequest: GET: /db?p=${p}&host=app-${hostIndex}&size=${size} - authority: app-${hostIndex}.{{ wildcard_domain }}:443 + authority: app-${hostIndex}.{{ wildcard_domain }} headers: x-variant: fromVar: variant @@ -128,4 +124,4 @@ phases: forks: simple: *simple proxy: *proxy - db: *db \ No newline at end of file + db: *db diff --git a/check-services.sh b/check-services.sh index d094882..92aa5f6 100755 --- a/check-services.sh +++ b/check-services.sh @@ -3,10 +3,10 @@ DOMAIN=$1 NUM_APPS=$2 -REPLICAS=$(oc get deployment -n mesh-control-plane istio-ingressgateway -o json | jq -r .spec.replicas) +REPLICAS=$(oc get deployment -n istio-system istio-ingressgateway -o json | jq -r .spec.replicas) echo "Waiting for all gateways to come up" while true; do - RUNNING=$(oc get po -n mesh-control-plane -l app=istio-ingressgateway --field-selector 'status.phase=Running' --no-headers | wc -l) + RUNNING=$(oc get po -n istio-system -l app=istio-ingressgateway --field-selector 'status.phase=Running' --no-headers | wc -l) if [ $RUNNING -eq $REPLICAS ]; then break; fi sleep 1; done @@ -28,5 +28,4 @@ if [ $STATUS = "ok" ]; then fi echo "Deleting ingress gateway" -oc delete po -n mesh-control-plane -l app=istio-ingressgateway -exit 1; \ No newline at end of file +exit 1; diff --git a/clusterrole-fix.yaml b/clusterrole-fix.yaml new file mode 100644 index 0000000..f6928fa --- /dev/null +++ b/clusterrole-fix.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-sidecar-injector-istio-system + labels: + app: istio-sidecar-injector + release: istio + istio: sidecar-injector +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "patch", "create" ] diff --git a/dc.yaml.j2 b/deployment.yaml.j2 similarity index 85% rename from dc.yaml.j2 rename to deployment.yaml.j2 index 545576f..f7e0813 100644 --- a/dc.yaml.j2 +++ b/deployment.yaml.j2 @@ -7,8 +7,8 @@ items: {% else %} {% set app_variant = "stable" %} {% endif %} -- apiVersion: apps.openshift.io/v1 - kind: DeploymentConfig +- apiVersion: apps/v1 + kind: Deployment metadata: labels: app: scale-test @@ -17,6 +17,14 @@ items: namespace: {{ namespace }} spec: replicas: {{ pods_per_dc }} + selector: + matchLabels: + app: scale-test + deploymentconfig: app-{{ index }} + app.index: application-{{ (index + 4) // 5 }} + app.variant: {{ app_variant }} + # Kiali would complain if there were no version label + version: "1.0" strategy: type: Recreate template: @@ -48,7 +56,7 @@ items: topologySpreadConstraints: - maxSkew: 1 topologyKey: workload - whenUnsatisfiable: DoNotSchedule + whenUnsatisfiable: ScheduleAnyway labelSelector: matchLabels: app: scale-test @@ -58,7 +66,16 @@ items: operator: Exists effect: NoExecute containers: - - imagePullPolicy: Always +{% if backend_type == "java-Z" %} + - image: quay.io/cfilleke/mannequin:0.0-z +{% endif %} +{% if backend_type == "java" %} + - image: quay.io/cfilleke/mannequin:0.0-z +{% endif %} +{% if backend_type == "native" %} + - image: quay.io/rvansa/mannequin:native +{% endif %} + imagePullPolicy: Always name: mannequin {% if backend_type is not defined or backend_type == "java" %} image: quay.io/rvansa/mannequin:jvm-latest diff --git a/dr-auth.yml b/dr-auth.yml new file mode 100644 index 0000000..91db2fc --- /dev/null +++ b/dr-auth.yml @@ -0,0 +1,10 @@ +apiVersion: "networking.istio.io/v1alpha3" +kind: "DestinationRule" +metadata: + name: "default" + namespace: "istio-system" +spec: + host: "*.local" + trafficPolicy: + tls: + mode: ISTIO_MUTUAL \ No newline at end of file diff --git a/generate-dc.yaml b/generate-dc.yaml index 8cc24a2..95c1b23 100644 --- a/generate-dc.yaml +++ b/generate-dc.yaml @@ -1,11 +1,11 @@ - name: Pick temp file tempfile: - prefix: dc- + prefix: deployment- suffix: .yaml register: resources - name: Generate deployment configs template: - src: dc.yaml.j2 + src: deployment.yaml.j2 dest: "{{ resources.path }}" - name: Apply resources command: "oc apply -f {{ resources.path }}" diff --git a/hosts.mysetup b/hosts.mysetup new file mode 100644 index 0000000..3cf3ea8 --- /dev/null +++ b/hosts.mysetup @@ -0,0 +1,72 @@ +[hyperfoil:children] +hyperfoil_controller +hyperfoil_agent + +[hyperfoil_controller] + +[hyperfoil_agent] +localhost + +[hyperfoil:vars] +hyperfoil_jfr=false + +[openshift] +localhost + +[openshift:vars] +proxy_cpu=200m +backend_cpu=300m +backend_type=java + +[compute] + +[all:vars] +oc_username=kubeadmin +#oc_token=sha256~THE-TOKEN +oc_password=THE-CLUSTER-PASSWORD +hyperfoil_deployer=k8s +hyperfoil_controller_protocol=https +hyperfoil_controller_host=hyperfoil.apps.test.perf-lab-myocp4.com +hyperfoil_controller_port=443 +hyperfoil_validate_certs=false +test_rampup_duration=120 +test_steady_duration=300 +test_name=closedmodel +#test_name=shardedwrk +fork=all +http2="true" + +# For open-model +test_users_per_sec=4500 +test_inc_users_per_sec=1500 +# For closed-model +#test_concurrent_users=12000 +#test_shared_connections=5376 +#test_shared_connections=60000 +test_concurrent_users=6000 +test_shared_connections=6000 + +pods_per_dc=1 +num_dc=40 +use_istio=true +#use_istio=false +record_envoy_stats=false +#namespace=plain-scale +#wildcard_domain=plain.apps.test.perf-lab-myocp4.com + +namespace=mesh-scale +control_plane=mesh-control-plane +wildcard_domain=mesh.apps.test.perf-lab-myocp4.com + +#namespace=istio-scale +#control_plane=istio-system +#wildcard_domain=istio.apps.test.perf-lab-myocp4.com + +policy_version=new + +target=gateways +#target=routers +#target=services + +expected_routers=0 +expected_gateways="1" diff --git a/hosts.quicklab b/hosts.quicklab new file mode 100644 index 0000000..65dd912 --- /dev/null +++ b/hosts.quicklab @@ -0,0 +1,74 @@ +[hyperfoil:children] +hyperfoil_controller +hyperfoil_agent + +[hyperfoil_controller] +#hfc ansible_host=f20-h22-000-r620.rdu2.scalelab.redhat.com ansible_user=root + +[hyperfoil_agent] +worker-2.scaletest.lab.psi.pnq2.redhat.com ansible_user=quicklab ansible_ssh_private_key_file=~/.ssh/quicklab.key +[hyperfoil:vars] +hyperfoil_jfr=false +#hyperfoil_log_config=/root/log4j2-trace-jgroups.xml +#hyperfoil_log_config=/home/benchuser/rvansa/log4j2-info.xml +#hyperfoil_controller_debug_port=0.0.0.0:5005 +#hyperfoil_agent_debug_port=0.0.0.0:5006 +#hyperfoil_agent_debug_suspend=y + +[openshift] +localhost + +[openshift:vars] +backend_cpu=300m +backend_type=java + +[compute] +#tbd + +[all:vars] +hyperfoil_distribution=/home/rvansa/workspace/Hyperfoil/distribution/target/distribution +hyperfoil_deployer=k8s +hyperfoil_controller_protocol=https +hyperfoil_controller_host=hyperfoil.apps.scaletest.lab.psi.pnq2.redhat.com +hyperfoil_controller_port=443 +hyperfoil_validate_certs=false +test_rampup_duration=120 +test_steady_duration=60 +test_name=closedmodel +#test_name=shardedwrk +fork=all +http2="true" +proxy_cpu=200m + +# For open-model +test_users_per_sec=4500 +test_inc_users_per_sec=1500 +# For closed-model +test_concurrent_users=600 +test_shared_connections=600 + +pods_per_dc=1 +num_dc=5 +use_istio=true +#use_istio=false +record_envoy_stats=false +#namespace=plain-scale +#wildcard_domain=plain.apps.ocp.scalelab + + +namespace=mesh-scale +control_plane=mesh-control-plane +wildcard_domain=mesh.apps.scaletest.lab.psi.pnq2.redhat.com + +#namespace=istio-scale +#control_plane=istio-system +#wildcard_domain=istio.apps.ocp.scalelab + +policy_version=new + +target=gateways +#target=routers +#target=services + +expected_routers=0 +expected_gateways="1" diff --git a/hosts.withossm b/hosts.withossm new file mode 100644 index 0000000..1675b91 --- /dev/null +++ b/hosts.withossm @@ -0,0 +1,73 @@ +[hyperfoil:children] +hyperfoil_controller +hyperfoil_agent + +[hyperfoil_controller] + +[hyperfoil_agent] +localhost + +[hyperfoil:vars] +hyperfoil_jfr=false + +[openshift] +localhost + +[openshift:vars] +proxy_cpu=200m +backend_cpu=300m +backend_type=java-Z + +[compute] + +[all:vars] +oc_username=kubeadmin +#oc_token=sha256~THE-TOKEN +oc_password=**** +hyperfoil_deployer=k8s +hyperfoil_controller_protocol=http +hyperfoil_controller_host=hyperfoil-hyperfoil.apps.ocp1.tt.testing +hyperfoil_controller_port=443 +hyperfoil_validate_certs=false +test_rampup_duration=10s +test_steady_duration=10s +test_name=closedmodel +#test_name=shardedwrk +fork=all +http2="true" +agent=agent-one + +# For open-model +test_users_per_sec=10 +test_inc_users_per_sec=5 +# For closed-model +#test_concurrent_users=12000 +#test_shared_connections=5376 +#test_shared_connections=60000 +test_concurrent_users=10 +test_shared_connections=10 + +pods_per_dc=1 +num_dc=5 +use_istio=true +#use_istio=false +record_envoy_stats=false +#namespace=plain-scale +#wildcard_domain=plain.apps.test.ocp1.tt.testing + +namespace=mesh-scale +#control_plane=mesh-control-plane +wildcard_domain=mesh.apps.test.ocp1.tt.testing + +#namespace=istio-scale +control_plane=istio-system +#wildcard_domain=istio.apps.test.ocp1.tt.testing + +policy_version=new + +target=gateways +#target=routers +#target=services + +expected_routers=0 +expected_gateways="1" diff --git a/hosts.withoutossm b/hosts.withoutossm new file mode 100644 index 0000000..5450fdb --- /dev/null +++ b/hosts.withoutossm @@ -0,0 +1,74 @@ +[hyperfoil:children] +hyperfoil_controller +hyperfoil_agent + +[hyperfoil_controller] + +[hyperfoil_agent] +localhost + +[hyperfoil:vars] +hyperfoil_jfr=false + +[openshift] +localhost + +[openshift:vars] +proxy_cpu=200m +backend_cpu=300m +backend_type=java-Z + +[compute] + +[all:vars] +oc_username=kubeadmin +#oc_token=sha256~THE-TOKEN +oc_password=**** +hyperfoil_deployer=k8s +hyperfoil_controller_protocol=http +hyperfoil_controller_host=hyperfoil-hyperfoil.apps.ocp1.tt.testing +hyperfoil_controller_port=443 +hyperfoil_validate_certs=false +test_rampup_duration=10s +test_steady_duration=10s +test_name=closedmodel-withoutossm +#test_name=shardedwrk +fork=all +http2="true" +agent=agent-one +single_node=true + +# For open-model +test_users_per_sec=10 +test_inc_users_per_sec=5 +# For closed-model +#test_concurrent_users=12000 +#test_shared_connections=5376 +#test_shared_connections=60000 +test_concurrent_users=10 +test_shared_connections=200 + +pods_per_dc=1 +num_dc=5 +#use_istio=true +use_istio=false +record_envoy_stats=false +#namespace=plain-scale +#wildcard_domain=plain.apps.test.ocp1.tt.testing + +namespace=mesh-scale +#control_plane=mesh-control-plane +wildcard_domain=mesh.apps.test.ocp1.a3elp50.lnxero1.boe + +#namespace=istio-scale +control_plane=istio-system +#wildcard_domain=istio.apps.test.ocp1.tt.testing + +policy_version=new + +#target=gateways +target=routers +#target=services + +expected_routers="1" +expected_gateways="0" diff --git a/java-runner.sh b/java-runner.sh new file mode 100644 index 0000000..795d3f3 --- /dev/null +++ b/java-runner.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash + +java -cp '/deployments/lib/*' -jar /deployments/app.jar diff --git a/loop-namespaces.sh b/loop-namespaces.sh new file mode 100755 index 0000000..c9d6942 --- /dev/null +++ b/loop-namespaces.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -e +set +x + +# This script performs setup & execution of load test over a cluster with a +# decreasing amount of namespaces. + +# This documentation assumes an up and running openshift install: +# I have worked up to 200 namespaces on my quicklab instance/install which consists of: +# 3 masters - 4cores/28GB, 3workers - 4cores/30GB + +# First, install hyperfoil through operatorhub. +# Then, create a namespace called "hyperfoil" and deploy hyperfoil to it +# NOTE: last time I have had to replace "hyperfoil.apps.mycloud.example.com" with the +# real wildcard postfix hyperfoil.apps.scaletest5.lab.upshift.rdu2.redhat.com in the yaml view of "create hyperfoil" +# Now update hyperfoil_agent in the hosts.scalelab file to point the the node that runs this hyperfoil pod/instance +# (otherwise the ' Upload benchmark template' step in test.yaml will fail) +# e.g. I ended up with +# worker-0.scaletest5.lab.upshift.rdu2.redhat.com ansible_user=quicklab ansible_ssh_private_key_file=~/.ssh/quicklab.key +# The quicklab.key file can be obtained from the quicklab cluster details web page. +# +# now assign roles for running the gateway (router) and workload (workload): + +# oc label node/worker-1.scaletest5.lab.upshift.rdu2.redhat.com test.role=router +# oc label node/worker-2.scaletest5.lab.upshift.rdu2.redhat.com test.role=workload + +# Ideally the load generator, workload and gateway are not competing for resources during test execution. +# This will make the gateway run on worker-1 and the app workload on worker-2. +# (if you don't do this, gateway deployment and app deployment may hang during test setup) + +# You may also need to set up ansible for hyperfoil, by running: +# $ ansible-galaxy collection install hyperfoil.hyperfoil_test +# $ ansible-galaxy install hyperfoil.hyperfoil_test +# $ ansible-galaxy install hyperfoil.hyperfoil_setup + + +# Install OSSM +VERSION=2.1 ./service-mesh-install.sh +# Set up the test prerequisites +ansible-playbook -v -i hosts.quicklab setup.yaml + +# Iterate over a decreasing number of namespaces +for num_namespaces in 10 0 ; do + ansible-playbook -e num_namespaces=$num_namespaces -v -i hosts.quicklab test.yaml +done + +# Clean up the mesh. +./service-mesh-delete.sh + diff --git a/oc-routes.yaml.j2 b/oc-routes.yaml.j2 new file mode 100644 index 0000000..b3d3834 --- /dev/null +++ b/oc-routes.yaml.j2 @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: List +items: +{% for index in range(1, (num_apps | int) + 1) %} +- apiVersion: route.openshift.io/v1 + kind: Route + metadata: + annotations: + haproxy.router.openshift.io/balance: roundrobin + haproxy.router.openshift.io/disable_cookies: "true" + haproxy.router.openshift.io/x-variant: stable + labels: + app: scale-test + name: app-{{ index }} + namespace: {{ namespace }} + spec: + host: app-{{ index }}.{{ wildcard_domain }} + port: + targetPort: http-8080 + tls: + termination: edge + to: + kind: Service + name: app-{{ index }} + weight: 100 + wildcardPolicy: None +{% endfor %} diff --git a/prep-nodes.sh b/prep-nodes.sh new file mode 100755 index 0000000..84ae09c --- /dev/null +++ b/prep-nodes.sh @@ -0,0 +1,14 @@ +#!/usr/bin/bash + +# need to label the nodes + +for i in `oc get no | grep worker | awk '{print $1}'` +do + oc label node $i test.role=workload + oc label node $i workload=${i} +done + +for i in `oc get no | grep worker | awk '{print $1}' | tail -1` +do + oc label node $i test.role=router --overwrite +done diff --git a/roles/.DS_Store b/roles/.DS_Store new file mode 100644 index 0000000..f249570 Binary files /dev/null and b/roles/.DS_Store differ diff --git a/roles/hyperfoil_test/benchmarks/example.yaml.j2 b/roles/hyperfoil_test/benchmarks/example.yaml.j2 new file mode 100644 index 0000000..d8d55a1 --- /dev/null +++ b/roles/hyperfoil_test/benchmarks/example.yaml.j2 @@ -0,0 +1,17 @@ +name: example +agents: + agent-one: + image: quay.io/ibm-z-performance/hyperfoil:0.25 + pod-cpu: "6500m" + pod-memory: "6Gi" +http: + host: {{ test_domain | default("http://hyperfoil.io", true) }} +phases: +- example: + atOnce: + users: 1 + scenario: + - test: + - httpRequest: + method: GET + path: / diff --git a/roles/hyperfoil_test/defaults/main.yml b/roles/hyperfoil_test/defaults/main.yml new file mode 100644 index 0000000..2fa84d4 --- /dev/null +++ b/roles/hyperfoil_test/defaults/main.yml @@ -0,0 +1,8 @@ +hyperfoil_agent_group: hyperfoil_agent +hyperfoil_agent_port: 22 +hyperfoil_controller_group: hyperfoil_controller +hyperfoil_controller_port: 8090 +hyperfoil_controller_protocol: http +hyperfoil_deployer: k8s +hyperfoil_validate_certs: false + diff --git a/roles/hyperfoil_test/meta/main.yml b/roles/hyperfoil_test/meta/main.yml new file mode 100644 index 0000000..d2257b7 --- /dev/null +++ b/roles/hyperfoil_test/meta/main.yml @@ -0,0 +1,20 @@ +galaxy_info: + author: Radim Vansa + description: Hyperfoil test runner + company: Red Hat + issue_tracker_url: https://github.com/Hyperfoil/hyperfoil-ansible/issues + license: Apache License, Version 2.0 + min_ansible_version: 2.4 + platforms: + - name: Fedora + versions: + - all + - name: EL + versions: + - all + galaxy_tags: + - hyperfoil + - benchmark + - loaddriver + - microservice +dependencies: [] diff --git a/roles/hyperfoil_test/tasks/main.yml b/roles/hyperfoil_test/tasks/main.yml new file mode 100644 index 0000000..ccac2bb --- /dev/null +++ b/roles/hyperfoil_test/tasks/main.yml @@ -0,0 +1,65 @@ +# - name: Gather information +# when: hyperfoil_controller_host is undefined +# block: +# - name: Validate that there's only single controller +# when: (groups[hyperfoil_controller_group] | length) != 1 +# fail: +# msg: Only one controller is allowed. +# - name: Gather facts about controller +# when: hostvars[groups[hyperfoil_controller_group][0]]['ansible_hostname'] is undefined +# setup: {} +# delegate_to: groups[hyperfoil_controller_group][0] +# delegate_facts: true +# - name: Set hyperfoil_controller_host +# set_fact: +# hyperfoil_controller_host: "{{ hostvars[groups[hyperfoil_controller_group][0]]['ansible_hostname'] }}" +# - name: Gather facts about agents +# when: hyperfoil_deployer == "" and hostvars[groups[hyperfoil_agent_group][0]]['ansible_hostname'] is undefined +# setup: {} +# delegate_to: "{{ item }}" +# delegate_facts: true +# loop: "{{ groups[hyperfoil_agent_group] }}" +- name: Apply benchmark template + template: + src: 'benchmarks/{{ test_name }}.yaml.j2' + dest: '/tmp/{{ test_name }}.yaml' + force: yes +- name: Define benchmark upload test file variable + set_fact: + benchmark_file: '/tmp/{{ test_name }}.yaml' +- name: Define optional upload test files variable + when: test_files is undefined + set_fact: + test_files: [] +- name: log vars + debug: + msg: "Validate: >{{ hyperfoil_validate_certs }}<" +- name: Upload benchmark template + shell: + "{{ lookup('template', 'templates/curl.j2') }} -L --post302 {{ '-k' if not (hyperfoil_validate_certs | bool) else '' }} \ + {{ hyperfoil_controller_protocol }}://{{ hyperfoil_controller_host }}/benchmark" + register: curl_cmd + failed_when: curl_cmd.rc > 1 +- name: Start benchmark + uri: + url: "{{ hyperfoil_controller_protocol }}://{{ hyperfoil_controller_host }}/benchmark/{{ test_name }}/start{{ ('?desc=' + run_description ) if run_description is defined else '' }}" + status_code: "202" + validate_certs: "{{ hyperfoil_validate_certs | bool }}" + register: test_run +- name: Set run ID + set_fact: + test_runid: "{{ test_run.location | urlsplit('path') | basename }}" +- name: Log run ID + debug: + msg: "Benchmark URI: {{ test_run.location }} -> {{ test_runid }}" +- name: Poll benchmark completion + uri: + url: "{{ hyperfoil_controller_protocol }}://{{ hyperfoil_controller_host }}/run/{{ test_runid }}/stats/recent" + validate_certs: "{{ hyperfoil_validate_certs | bool }}" + return_content: true + register: test_status + # Ansible 2.7 does not support include_tasks in do-until loop so we need to inline the condition + # See https://github.com/ansible/ansible/issues/46203 for details + until: test_status.json.status == "TERMINATED" + delay: 5 + retries: 100000 diff --git a/roles/hyperfoil_test/templates/curl.j2 b/roles/hyperfoil_test/templates/curl.j2 new file mode 100644 index 0000000..d092cce --- /dev/null +++ b/roles/hyperfoil_test/templates/curl.j2 @@ -0,0 +1 @@ +curl -f -X POST -F "benchmark=@{{ benchmark_file }};filename={{ benchmark_file }}" {% for upload in test_files %} -F "file{{loop.index}}=@{{upload}};filename={{upload}}" {% endfor %} \ No newline at end of file diff --git a/routing.yaml.j2 b/routing.yaml.j2 index 690a7d0..0740f74 100644 --- a/routing.yaml.j2 +++ b/routing.yaml.j2 @@ -185,4 +185,14 @@ items: portLevelMtls: 5432: mode: DISABLE -{% endif %} \ No newline at end of file +{% endif %} +- apiVersion: "networking.istio.io/v1alpha3" + kind: "DestinationRule" + metadata: + name: "default" + namespace: {{ control_plane }} + spec: + host: "*.local" + trafficPolicy: + tls: + mode: ISTIO_MUTUAL \ No newline at end of file diff --git a/service-mesh-delete.sh b/service-mesh-delete.sh index b52755e..5d161eb 100755 --- a/service-mesh-delete.sh +++ b/service-mesh-delete.sh @@ -29,6 +29,7 @@ oc delete validatingwebhookconfiguration openshift-operators.servicemesh-resourc # Operator itself seems to persist oc delete deployment -n openshift-operators istio-operator +oc delete daemonset -n openshift-operators istio-cni-node oc delete daemonset -n openshift-operators istio-node istio-cni-node oc delete sa -n openshift-operators istio-operator oc delete sa -n openshift-operators istio-cni diff --git a/service-mesh-install.sh b/service-mesh-install.sh index 85518b5..fbcbc9a 100755 --- a/service-mesh-install.sh +++ b/service-mesh-install.sh @@ -35,7 +35,7 @@ do done fi if [ $((RETRY%12)) -eq 11 ]; then - oc delete csv kiali-operator.v1.36.6 -n openshift-operators + oc delete csv kiali-operator.v1.36.6 -n openshift-operators || true fi sleep 5 RETRY=$((RETRY + 1)) diff --git a/set_number_of_namespaces.sh b/set_number_of_namespaces.sh new file mode 100755 index 0000000..43d0988 --- /dev/null +++ b/set_number_of_namespaces.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +NUM_NS=${1:-0} +MEASURE_PERIOD=${MEASURE_PERIOD:-120} + +NS_PREFIX=foobar-ns-perftest +EXISTING_PROJECTS=( $(oc projects --short | grep $NS_PREFIX) ) +EXPECTED_PROJECTS=() +CREATED_PROJECTS=() +DELETED_PROJECTS=() + +set -e +# set -x + +function sample_stats() { + echo "sample stats for $2 seconds" + sleep $2 + echo "get and store statistics" + mkdir -p sampled_stats/ + pushd sampled_stats > /dev/null + START=$(date +%s) + END=$(( $START+ $2*1000 )) + STEP="60" + # cpu + oc exec -n openshift-monitoring prometheus-k8s-0 -c prometheus -it -- curl "http://127.0.0.1:9090/api/v1/query_range?start=${START}&end=${END}&step=${STEP}&query=%0A++++++sum%28%0A++++++++kube_pod_resource_request%7Bresource%3D%22cpu%22%7D%0A++++++++*%0A++++++++on%28node%29+group_left%28role%29+%28%0A++++++++++max+by+%28node%29+%28kube_node_role%7Brole%3D%7E%22.%2B%22%7D%29%0A++++++++%29%0A++++++%29%0A++++" > $1_cpu.json + # memory + oc exec -n openshift-monitoring prometheus-k8s-0 -c prometheus -it -- curl "http://127.0.0.1:9090/api/v1/query_range?start=${START}&end=${END}&step=${STEP}&query=%0A++++++sum%28%0A++++++++%28node_memory_MemTotal_bytes+-+node_memory_MemAvailable_bytes%29%0A++++++++*%0A++++++++on%28instance%29+group_left%28role%29+%28%0A++++++++++label_replace%28max+by+%28node%29+%28kube_node_role%7Brole%3D%7E%22.%2B%22%7D%29%2C+%22instance%22%2C+%22%241%22%2C+%22node%22%2C+%22%28.*%29%22%29%0A++++++++%29%0A++++++%29%0A++++" > $1_memory.json + # network rx bandwidth rx + oc exec -n openshift-monitoring prometheus-k8s-0 -c prometheus -it -- curl "http://127.0.0.1:9090/api/v1/query_range?start=${START}&end=${END}&step=${STEP}&query=%0A++++++sum%28%0A++++++++instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%0A++++++++*%0A++++++++on%28instance%29+group_left%28role%29+%28%0A++++++++++label_replace%28max+by+%28node%29+%28kube_node_role%7Brole%3D%7E%22.%2B%22%7D%29%2C+%22instance%22%2C+%22%241%22%2C+%22node%22%2C+%22%28.*%29%22%29%0A++++++++%29%0A++++++%29%0A++++" > $1_bandwidth_rx.json + # network tx bandwith + oc exec -n openshift-monitoring prometheus-k8s-0 -c prometheus -it -- curl "http://127.0.0.1:9090/api/v1/query_range?start=${START}&end=${END}&step=${STEP}&query=%0A++++++sum%28%0A++++++++instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%0A++++++++*%0A++++++++on%28instance%29+group_left%28role%29+%28%0A++++++++++label_replace%28max+by+%28node%29+%28kube_node_role%7Brole%3D%7E%22.%2B%22%7D%29%2C+%22instance%22%2C+%22%241%22%2C+%22node%22%2C+%22%28.*%29%22%29%0A++++++++%29%0A++++++%29%0A++++" > $1_bandwidth_tx.json + popd > /dev/null +} + +oc whoami > /dev/null + +echo "Ensuring that there will be ${NUM_NS} namespaces." +cp smmr.yaml smmr_with_added_namespaces.yaml + +for ((i=0;i<$NUM_NS;i++)) +do + EXPECTED_PROJECTS+=($NS_PREFIX-${i}) + echo " - $NS_PREFIX-${i}" >> smmr_with_added_namespaces.yaml +done + +for value in "${EXISTING_PROJECTS[@]}" +do + if [[ ! " ${EXPECTED_PROJECTS[*]} " =~ " ${value} " ]]; then + echo "delete namespace ${value}" + DELETED_PROJECTS+=(${value}) + oc delete pod --all --namespace="${value}" && oc delete project "${value}" & + else + echo "namespace ${value} already exists, not deleting it" + fi +done + + +for value in "${EXPECTED_PROJECTS[@]}" +do + if [[ ! " ${EXISTING_PROJECTS[*]} " =~ " ${value} " ]]; then + echo "didn't find expected project ${value}" + CREATED_PROJECTS+=(${value}) + # we need to delay launches in order to avoid being rate limited, it seems. + sleep 1 + oc new-project --skip-config-write=true "${value}" && oc create --namespace "${value}" -f smallsleepingpod.yaml & + else + echo "expected namespace ${value} already exists, not creating it" + fi +done + +echo "projects updated, wait for background processes (if any)" +wait < <(jobs -p) +echo "projects updated, wait for project deletion to complete" + +for value in "${DELETED_PROJECTS[@]}" + do + echo "wait for deletion of namespace ${value} to complete" + until [ $( oc projects --short | grep "${value}" | wc -l ) == 0 ] + do + echo -n . + done +done + +for value in "${CREATED_PROJECTS[@]}" +do + echo "wait for newly created namespace ${value} to ready up" + oc wait --for=condition=Ready pods --all --namespace="${value}" +done + +# Include the current set of namespaces in the mesh, and wait for it to catch up +oc apply -f smmr_with_added_namespaces.yaml +oc wait --for condition=Ready -n mesh-control-plane smcp/basic-install --timeout 1800s +oc wait --for condition=Ready -n mesh-control-plane smmr/default --timeout 1800s + +if [ $MEASURE_PERIOD != "0" ]; then + sample_stats "$NUM_NS" "$MEASURE_PERIOD" +fi + +echo "done" +rm smmr_with_added_namespaces.yaml || true +exit 0 diff --git a/setup.yaml b/setup.yaml index 7d477a7..3a0f8a2 100644 --- a/setup.yaml +++ b/setup.yaml @@ -1,12 +1,13 @@ - hosts: openshift + connection: local gather_facts: false tasks: - name: Set number of apps set_fact: num_apps: "{{ (num_dc | int) / 5 }}" - - name: Login - when: oc_username is defined and oc_password is defined - command: "oc login --username={{ oc_username }} --password={{ oc_password }}" + #- name: Login + # when: oc_username is defined and oc_password is defined + # command: "oc login --username={{ oc_username }} --password={{ oc_password }} --server=api.ocp1.a3elp50.lnxero1.boe:6443" - name: Check namespace presence command: "oc get project {{ namespace }}" register: ns_exists @@ -82,13 +83,24 @@ - name: Drop existing routes command: "oc delete route --all -n {{ namespace }}" failed_when: false - - name: Expose route - command: "oc create route reencrypt app-{{ item }} --cert=/home/rvansa/workspace/mannequin/src/main/resources/domain.crt --key=/home/rvansa/workspace/mannequin/src/main/resources/domain.key --service=app-{{ item }} --port=8443 --hostname=app-{{ item }}.{{ wildcard_domain }}" - loop: "{{ range(1, (num_apps | int) + 1, 1) | list }}" + - name: Get temporary route files + tempfile: + prefix: oc-routes- + suffix: .yaml + register: ocroutes + - name: Generate route CRs + template: + src: oc-routes.yaml.j2 + dest: "{{ ocroutes.path }}" + - name: Apply routes + command: "oc apply -f {{ ocroutes.path }}" + #- name: Expose route + # command: "oc expose svc app-{{ item }}" + # loop: "{{ range(1, (num_apps | int) + 1, 1) | list }}" - name: Generate deployment configs include_tasks: generate-dc.yaml - name: Wait for pods to come up shell: "{{ playbook_dir }}/wait-for-pods.sh {{ namespace }} \"{{ pod_pattern }}\" {{ num_dc }}" -- hosts: hyperfoil_controller - roles: - - hyperfoil.hyperfoil_setup +# - hosts: hyperfoil_controller +# roles: +# - hyperfoil.hyperfoil_setup diff --git a/sidecar-injector.yaml b/sidecar-injector.yaml new file mode 100644 index 0000000..f6928fa --- /dev/null +++ b/sidecar-injector.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-sidecar-injector-istio-system + labels: + app: istio-sidecar-injector + release: istio + istio: sidecar-injector +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "patch", "create" ] diff --git a/smallsleepingpod.yaml b/smallsleepingpod.yaml new file mode 100644 index 0000000..2349251 --- /dev/null +++ b/smallsleepingpod.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: myapp-pod + labels: + app: myapp +spec: + containers: + - name: myapp-container + image: envoyproxy/nighthawk-dev:latest + command: ['sh', '-c', 'echo The app is running! && sleep 3600'] diff --git a/smcp_v2.yaml b/smcp_v2.yaml index 88a5d22..2bffd8d 100644 --- a/smcp_v2.yaml +++ b/smcp_v2.yaml @@ -41,14 +41,17 @@ spec: cpu: "256" memory: 256Gi requests: - cpu: "16" + cpu: "7" memory: 2Gi deployment: autoScaling: enabled: false pod: nodeSelector: - ${INGRESS_NODE_SELECTOR} + test.role: router + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists service: {} openshiftRoute: enabled: true diff --git a/test.yaml b/test.yaml index bb7127b..351cd88 100644 --- a/test.yaml +++ b/test.yaml @@ -1,23 +1,27 @@ -# Gather facts -- hosts: hyperfoil - ignore_unreachable: true - tasks: [] -- hosts: hyperfoil_controller - tasks: - - name: Start test - include_role: - name: hyperfoil.hyperfoil_test - when: hyperfoil_deployer == "ssh" - vars: - test_name: test - hosts: openshift + connection: local tasks: - set_fact: http2: "{{ http2 | default('true', true) }}" fork: "{{ fork | default('all', true) }}" num_apps: "{{ (num_dc | int) // 5 }}" - record_envoy_stats: "{{ record_envoy_stats | default('false', true) | bool }}" - use_istio: "{{ use_istio | default('true', true) | bool }}" + - name: Check if oc is logged on + shell: "oc whoami" + register: "login_check_cmd" + - name: Find workload nodes + shell: "oc get node -l test.role=workload -o json | jq -r '.items[].metadata.name'" + register: workload_cmd + - name: Get workload pod count + loop: "{{ workload_cmd.stdout.split() }}" + shell: "oc get pod -n {{ namespace }} --field-selector=spec.nodeName={{item}},status.phase=Running -o json | jq '.items | length'" + register: workload_counts +# - name: Calculate pod count difference + # set_fact: + # min_pods: "{{ workload_counts.results | map(attribute='stdout') | list | min }}" + # max_pods: "{{ workload_counts.results | map(attribute='stdout') | list | max }}" + # total_pods: "{{ workload_counts.results | map(attribute='stdout') | list | map('int') | sum }}" + # record_envoy_stats: "{{ record_envoy_stats | default('false', true) | bool }}" + #use_istio: "{{ use_istio | default('true', true) | bool }}" - when: single_node is not defined block: - name: Find workload nodes @@ -50,12 +54,12 @@ when: "(max_pods | int) - (min_pods | int) > 1" fail: msg: "Unbalanced pods: min = {{ min_pods }}, max = {{ max_pods }} " - - name: Make sure all services are reachable - shell: "{{ playbook_dir }}/check-services.sh {{ wildcard_domain }} {{ num_apps }}" - register: services_result - until: services_result.rc == 0 - retries: 10 - delay: 10 +# - name: Make sure all services are reachable +# shell: "{{ playbook_dir }}/check-services.sh {{ wildcard_domain }} {{ num_apps }}" +# register: services_result +# until: services_result.rc == 0 +# retries: 10 +# delay: 10 - when: target == 'routers' block: - name: Find routers @@ -102,6 +106,9 @@ when: use_istio shell: "oc get po -n {{ control_plane }} -l istio=ingressgateway -o name --no-headers | cut -f 2 -d '/'" register: gateway_pods + # - name: Create additional namespaces + # shell: "./set_number_of_namespaces.sh {{ num_namespaces }} 0" + # register: create_namespaces - name: Get workload pod names shell: "oc get po -n {{ namespace }} -l app=scale-test -o name --no-headers | cut -f 2 -d '/'" register: workload_pods @@ -113,19 +120,20 @@ poll: 0 register: stats_reset - name: Wait for the info - when: use_istio and record_envoy_stats + when: use_istio and record_envoy_stats | bool loop: "{{ stats_reset.results }}" async_status: jid: "{{ item.ansible_job_id }}" - name: Start test include_role: - name: hyperfoil.hyperfoil_test + name: hyperfoil_test when: hyperfoil_deployer == "k8s" vars: # run_description: "{{ namespace }}%20R%3D{{ num_routers }}%20G%3D{{ num_gateways }}%20P%3D{{ total_pods }}%20VS%3D{{ virtualservices.stdout }}" +# run_description: "sm2.1%20G%3D{{ num_gateways }}%20P%3D{{ total_pods }}%20N%3D{{ num_namespaces }}%20{{ fork }}%20{{ (http2 == 'true') | ternary('http2', 'http1.1') }}" run_description: "sm2.1%20G%3D{{ num_gateways }}%20P%3D{{ total_pods }}%20{{ fork }}%20{{ (http2 == 'true') | ternary('http2', 'http1.1') }}" - name: Capture info after test - when: use_istio and record_envoy_stats + when: use_istio and record_envoy_stats | bool include_tasks: capture-envoy-info.yaml vars: dest: "/tmp/envoy_{{test_runid}}_r{{ num_routers }}_g{{ num_gateways }}_d{{ num_dc }}.zip" @@ -133,4 +141,4 @@ debug: msg: "Run ID was {{ test_runid }}" - name: Fetch Hyperfoil stats - shell: "curl -s -k {{ hyperfoil_controller_protocol }}://{{ hyperfoil_controller_host }}:{{ hyperfoil_controller_port}}/run/{{ test_runid }}/stats/all -H 'accept: application/json' > /tmp/hf_{{ test_runid }}_r{{ num_routers }}_g{{ num_gateways }}_d{{ num_dc }}.json" + shell: "curl -s -k -L {{ hyperfoil_controller_protocol }}://{{ hyperfoil_controller_host }}/run/{{ test_runid }}/stats/all -H 'accept: application/json' > /tmp/hf_{{ test_runid }}_r{{ num_routers }}_g{{ num_gateways }}_d{{ num_dc }}.json"