From e25e7d7c5089973adf579003788b6bf3a348cfd6 Mon Sep 17 00:00:00 2001 From: cfillekes Date: Tue, 10 May 2022 07:42:39 -0400 Subject: [PATCH 1/5] changes necessary to run on IBM System Z --- prep-nodes.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 prep-nodes.sh diff --git a/prep-nodes.sh b/prep-nodes.sh new file mode 100755 index 0000000..84ae09c --- /dev/null +++ b/prep-nodes.sh @@ -0,0 +1,14 @@ +#!/usr/bin/bash + +# need to label the nodes + +for i in `oc get no | grep worker | awk '{print $1}'` +do + oc label node $i test.role=workload + oc label node $i workload=${i} +done + +for i in `oc get no | grep worker | awk '{print $1}' | tail -1` +do + oc label node $i test.role=router --overwrite +done From ca2bfa177a02667d0b50685de90edfae48a4be56 Mon Sep 17 00:00:00 2001 From: cfillekes Date: Tue, 10 May 2022 07:43:50 -0400 Subject: [PATCH 2/5] changes necessary to run on IBM System Z --- clusterrole-fix.yaml | 15 +++++++++ hosts.mysetup | 72 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 clusterrole-fix.yaml create mode 100644 hosts.mysetup diff --git a/clusterrole-fix.yaml b/clusterrole-fix.yaml new file mode 100644 index 0000000..f6928fa --- /dev/null +++ b/clusterrole-fix.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-sidecar-injector-istio-system + labels: + app: istio-sidecar-injector + release: istio + istio: sidecar-injector +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "patch", "create" ] diff --git a/hosts.mysetup b/hosts.mysetup new file mode 100644 index 0000000..3cf3ea8 --- /dev/null +++ b/hosts.mysetup @@ -0,0 +1,72 @@ +[hyperfoil:children] +hyperfoil_controller +hyperfoil_agent + +[hyperfoil_controller] + +[hyperfoil_agent] +localhost + +[hyperfoil:vars] +hyperfoil_jfr=false + +[openshift] +localhost + +[openshift:vars] +proxy_cpu=200m +backend_cpu=300m +backend_type=java + +[compute] + +[all:vars] +oc_username=kubeadmin +#oc_token=sha256~THE-TOKEN +oc_password=THE-CLUSTER-PASSWORD +hyperfoil_deployer=k8s +hyperfoil_controller_protocol=https +hyperfoil_controller_host=hyperfoil.apps.test.perf-lab-myocp4.com +hyperfoil_controller_port=443 +hyperfoil_validate_certs=false +test_rampup_duration=120 +test_steady_duration=300 +test_name=closedmodel +#test_name=shardedwrk +fork=all +http2="true" + +# For open-model +test_users_per_sec=4500 +test_inc_users_per_sec=1500 +# For closed-model +#test_concurrent_users=12000 +#test_shared_connections=5376 +#test_shared_connections=60000 +test_concurrent_users=6000 +test_shared_connections=6000 + +pods_per_dc=1 +num_dc=40 +use_istio=true +#use_istio=false +record_envoy_stats=false +#namespace=plain-scale +#wildcard_domain=plain.apps.test.perf-lab-myocp4.com + +namespace=mesh-scale +control_plane=mesh-control-plane +wildcard_domain=mesh.apps.test.perf-lab-myocp4.com + +#namespace=istio-scale +#control_plane=istio-system +#wildcard_domain=istio.apps.test.perf-lab-myocp4.com + +policy_version=new + +target=gateways +#target=routers +#target=services + +expected_routers=0 +expected_gateways="1" From f0c0744eebc0197180257eeef0eb0090ffb54d7a Mon Sep 17 00:00:00 2001 From: cfillekes Date: Tue, 10 May 2022 08:01:10 -0400 Subject: [PATCH 3/5] Dockerfile and entrypoint script for re-based mannequin imate --- Dockerfile | 27 +++++++++++++++++++++++++++ java-runner.sh | 3 +++ 2 files changed, 30 insertions(+) create mode 100644 Dockerfile create mode 100644 java-runner.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8baeeed --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +#### +# This Dockerfile is used in order to build a container that runs a +# Quarkus application in JVM mode on s390x architecture (IBM System Z) +# +# Before building the docker image run: +# +# mvn package +# +# Then, build the image with: +# +# podman build -f src/main/docker/Dockerfile.jvmZ -t quay.io//mannequin:0.0-z . +# +# Then run the container using: +# +# podman run -i --rm -p 8080:8080 quarkus/mannequin-jvm quay.io//mannequin:0.0-z . +# +# the instructions here are to put it in a public registry (e.g. quay.io) +# so that you can pull it into a pod deployment on OpenShift +# +### +FROM registry.access.redhat.com/ubi8/ubi:latest +RUN dnf -y update && dnf -y install java-1.8.0-openjdk-devel +ENV JAVA_OPTIONS=-Dquarkus.http.host=0.0.0.0 +COPY target/lib/* /deployments/lib/ +COPY target/*-runner.jar /deployments/app.jar +COPY ./java-runner.sh /deployments/java-runner.sh +ENTRYPOINT [ "/deployments/java-runner.sh" ] diff --git a/java-runner.sh b/java-runner.sh new file mode 100644 index 0000000..795d3f3 --- /dev/null +++ b/java-runner.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash + +java -cp '/deployments/lib/*' -jar /deployments/app.jar From e470e014214e7faa84290c7539f5060a5cb84767 Mon Sep 17 00:00:00 2001 From: cfillekes Date: Wed, 11 May 2022 08:26:54 -0400 Subject: [PATCH 4/5] create a yaml file for the sidecar injector role --- sidecar-injector.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 sidecar-injector.yaml diff --git a/sidecar-injector.yaml b/sidecar-injector.yaml new file mode 100644 index 0000000..f6928fa --- /dev/null +++ b/sidecar-injector.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-sidecar-injector-istio-system + labels: + app: istio-sidecar-injector + release: istio + istio: sidecar-injector +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "patch", "create" ] From d97d86cda703ea41b25c7a73c6b29079c3d23c24 Mon Sep 17 00:00:00 2001 From: cfillekes Date: Wed, 11 May 2022 14:00:42 -0400 Subject: [PATCH 5/5] reduce cpu allocation --- README.md | 49 +++++++++++++++++++++++++------------------------ dc.yaml.j2 | 5 ++++- setup.yaml | 2 +- smcp_v2.yaml | 2 +- 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index ab7580b..378ae7c 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,33 @@ # Istio/Maistra scalability tests -To get this benchmark running: +## Prerequisites -1. Install OCP and Ansible -2. Login to OCP: `oc login -u system:admin` -3. Install Istio: https://maistra.io/docs/getting_started/install/ +### OCP +### Ansible +### Hyperfoil +### Hyperfoil Ansible task + +```bash +git clone https://github.com/Hyperfoil/hyperfoil_setup.git +cd hyperfoil_setup +ansible-galaxy install hyperfoil.hyperfoil_setup +ansible-galaxy install hyperfoil.hyperfoil_shutdown +ansible-galaxy install hyperfoil.hyperfoil_test +``` + +### Firewall + +## Setup + +1. Install prerequisites +2. Run `prep_nodes.sh` to label the nodes. +3. Login to OCP: `oc login -u system:admin` +4. Install Istio: https://maistra.io/docs/getting_started/install/ - In `controlplane/basic-install` set `gateways.ior_enabled: true` and `mixer.telemetry.enabled: false` - I suggest locating `istio-system` pods on the infra node (the same where the `default/router` resides): `oc patch namespace istio-system -p '{"metadata":{"annotations":{"openshift.io/node-selector":"node-role.kubernetes.io/infra=true"}}}'` - - I had trouble with `clusterrole istio-sidecar-injector-istio-system` - this was not correctly created and I had to fix it manually, applying: -``` - apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-sidecar-injector-istio-system - labels: - app: istio-sidecar-injector - release: istio - istio: sidecar-injector -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch"] -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations"] - verbs: ["get", "list", "watch", "patch", "create" ] -``` -4. You might need to add the policies: + +5. You might need to add the policies: ``` oc adm policy add-scc-to-user anyuid -z istio-ingress-service-account -n istio-system oc adm policy add-scc-to-user anyuid -z default -n istio-system @@ -61,4 +62,4 @@ rules: * There seems to be a bug in IOR (MAISTRA-356) that is not resolved in the image I use. Therefore you have to manually fix the generated route: `oc get route -n istio-system -l maistra.io/generated-by=ior` `oc patch route -n istio-system app-gateway-xxxxx -p '{ "spec": { "port" : { "targetPort": 443 }}}'` TODO -oc get deployment istio-ingressgateway -o json | jq '.spec.template.spec.containers[].resources.requests={},.spec.template.spec.containers[].args += ["--proxy-concurrency", "4"]' \ No newline at end of file +oc get deployment istio-ingressgateway -o json | jq '.spec.template.spec.containers[].resources.requests={},.spec.template.spec.containers[].args += ["--proxy-concurrency", "4"]' diff --git a/dc.yaml.j2 b/dc.yaml.j2 index 299918f..d19b788 100644 --- a/dc.yaml.j2 +++ b/dc.yaml.j2 @@ -56,8 +56,11 @@ items: operator: Exists effect: NoExecute containers: +{% if backend_type == "java-Z" %} + - image: quay.io/cfilleke/mannequin:0.0-z +{% endif %} {% if backend_type == "java" %} - - image: quay.io/rvansa/mannequin:jvm-latest + - image: quay.io/cfilleke/mannequin:0.0-z {% endif %} {% if backend_type == "native" %} - image: quay.io/rvansa/mannequin:native diff --git a/setup.yaml b/setup.yaml index 7d477a7..240eedd 100644 --- a/setup.yaml +++ b/setup.yaml @@ -6,7 +6,7 @@ num_apps: "{{ (num_dc | int) / 5 }}" - name: Login when: oc_username is defined and oc_password is defined - command: "oc login --username={{ oc_username }} --password={{ oc_password }}" + command: "oc login --username={{ oc_username }} --password={{ oc_password }} --server=api.maistra-qez-49.maistra.upshift.redhat.com:6443" - name: Check namespace presence command: "oc get project {{ namespace }}" register: ns_exists diff --git a/smcp_v2.yaml b/smcp_v2.yaml index e88dadc..71079d9 100644 --- a/smcp_v2.yaml +++ b/smcp_v2.yaml @@ -41,7 +41,7 @@ spec: cpu: "256" memory: 256Gi requests: - cpu: "16" + cpu: "7" memory: 2Gi deployment: autoScaling: