latest changes for S390x

.gitignore

@@ -1,2 +1,3 @@
 results
 istioctl*
+sampled_stats/*

DeploymentExample.yml

@@ -0,0 +1,101 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: scale-test
+    version: "1.0"
+  name: app-2-n
+  namespace: mesh-scale
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: scale-test
+      app.index: application-1
+      app.variant: stable
+      deploymentconfig: app-2
+      version: "1.0"
+  strategy:
+    activeDeadlineSeconds: 21600
+    recreateParams:
+      timeoutSeconds: 600
+    resources: {}
+    type: Recreate
+  template:
+    metadata:
+      annotations:
+        sidecar.istio.io/concurrency: "1"
+        sidecar.istio.io/inject: "true"
+        sidecar.istio.io/proxyCPU: 200m
+        sidecar.istio.io/proxyCPULimit: 1000m
+        sidecar.istio.io/statsInclusionPrefixes: listener,cluster.outbound
+      creationTimestamp: null
+      labels:
+        app: scale-test
+        app.index: application-1
+        app.variant: stable
+        deploymentconfig: app-2
+        version: "1.0"
+    spec:
+      containers:
+      - env:
+        - name: NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        - name: VARIANT
+          value: canary
+        - name: JAVA_OPTIONS
+          value: |
+            -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.Log4j2LogDelegateFactory
+        image: quay.io/cfilleke/mannequin:0.0-z
+        imagePullPolicy: Always
+        name: mannequin
+        ports:
+        - containerPort: 8080
+          protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - curl
+            - localhost:8080/
+          failureThreshold: 3
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          requests:
+            cpu: 300m
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /var/certs
+          name: certs
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        test.role: workload
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - effect: NoExecute
+        key: workload
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app: scale-test
+        maxSkew: 1
+        topologyKey: workload
+        whenUnsatisfiable: ScheduleAnyway
+      volumes:
+      - name: certs
+        secret:
+          defaultMode: 420
+          secretName: app-1
+  test: false
+  triggers:
+  - type: ConfigChange

Dockerfile

@@ -0,0 +1,28 @@
+####
+# This Dockerfile is used in order to build a container that runs a 
+# Quarkus application in JVM mode on s390x architecture (IBM System Z)
+#
+# Before building the docker image run:
+#
+# mvn package
+#
+# Then, build the image with:
+#
+# podman build -f Dockerfile.Z -t quay.io/<username>/mannequin:0.0-z .
+#
+# Then run the container using:
+#
+# podman run -i --rm -p 8080:8080 quarkus/mannequin-jvm quay.io/<username>/mannequin:0.0-z .
+#
+# the instructions here are to put it in a public registry (e.g. quay.io)
+# so that you can pull it into a pod deployment on OpenShift
+#
+###
+FROM registry.access.redhat.com/ubi9/ubi:latest
+RUN yum install -y java-11-openjdk.s390x
+ENV JAVA_OPTIONS=-Dquarkus.http.host=0.0.0.0
+COPY target/lib/* /deployments/lib/
+COPY target/*-runner.jar /deployments/app.jar
+COPY ./java-runner.sh /deployments/java-runner.sh
+RUN chmod a+x /deployments/java-runner.sh
+ENTRYPOINT [ "/deployments/java-runner.sh" ]

README.md

@@ -1,53 +1,39 @@
 # Istio/Maistra scalability tests
 
-To get this benchmark running:
+##  Prerequisites
 
-1. Install OCP and Ansible
-2. Login to OCP: `oc login -u system:admin`
-3. Install Istio: https://maistra.io/docs/getting_started/install/
-    - In `controlplane/basic-install` set `gateways.ior_enabled: true` and `mixer.telemetry.enabled: false`
-    - I suggest locating `istio-system` pods on the infra node (the same where the `default/router` resides):
-      `oc patch namespace istio-system -p '{"metadata":{"annotations":{"openshift.io/node-selector":"node-role.kubernetes.io/infra=true"}}}'`
-    - I had trouble with `clusterrole istio-sidecar-injector-istio-system` - this was not correctly created and I had to fix it manually, applying:
-```
-      apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: istio-sidecar-injector-istio-system
-  labels:
-    app: istio-sidecar-injector
-    release: istio
-    istio: sidecar-injector
-rules:
-- apiGroups: [""]
-  resources: ["configmaps"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["admissionregistration.k8s.io"]
-  resources: ["mutatingwebhookconfigurations"]
-  verbs: ["get", "list", "watch", "patch", "create" ]
-```
-4. You might need to add the policies:
-   ```
-   oc adm policy add-scc-to-user anyuid -z istio-ingress-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z default -n istio-system
-   oc adm policy add-scc-to-user anyuid -z prometheus -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-egressgateway-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-citadel-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-ingressgateway-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-cleanup-old-ca-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-mixer-post-install-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-mixer-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-pilot-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-sidecar-injector-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-galley-service-account -n istio-system
-   oc adm policy add-scc-to-user anyuid -z istio-security-post-install-account -n istio-system
-   ```
-5. Allow wildcard routes: `oc set env dc/router ROUTER_ALLOW_WILDCARD_ROUTES=true -n default` (not possible in OCP 4.1)
-6. Create hosts.* according to your system
-7. Run the setup (now everything should be automatized):
-    `ansible-playbook -i hosts.mysetup setup.yaml`
-8. Start the test:
-    `ansible-playbook -i hosts.mysetup test.yaml`
+### 1. OCP 
+### 2. Ansible
+### 3. Hyperfoil
+
+https://github.ibm.com/Openshift-Addons-Performance/Openshift-performance-Docs-IBM-Z
+
+### 4. Firewall
+
+## Setup
+
+1. Install prerequisites
+2. Run `prep_nodes.sh` to label the nodes.
+3. Login to OCP: `oc login -u system:admin`
+
+## Test with OSSM
+    1. Install OSSM
+        - In `controlplane/basic-install` set `gateways.ior_enabled: true` and `mixer.telemetry.enabled: false`
+        - I suggest locating `istio-system` pods on the infra node (the same where the `default/router` resides):
+          `oc patch namespace istio-system -p '{"metadata":{"annotations":{"openshift.io/node-selector":"node-role.kubernetes.io/infra=true"}}}'`
+    2. Create hosts.* according to your system
+    3. Run the setup (now everything should be automatized):
+        `ansible-playbook -i hosts.withossm setup.yaml`
+    4. Start the test:
+        `ansible-playbook -i hosts.withossm test.yaml`
+
+## Test without OSSM
+    1. Create hosts.* according to your system
+    2. Run the setup (now everything should be automatized):
+        `ansible-playbook -i hosts.withoutossm setup.yaml`
+    3. Start the test:
+        `ansible-playbook -i hosts.withoutossm test.yaml`
+
 
 ## Hints:
 
@@ -61,4 +47,4 @@ rules:
 * There seems to be a bug in IOR (MAISTRA-356) that is not resolved in the image I use. Therefore you have to manually fix the generated route: `oc get route -n istio-system -l maistra.io/generated-by=ior` `oc patch route -n istio-system app-gateway-xxxxx -p '{ "spec": { "port" : { "targetPort": 443 }}}'`
 
 TODO
-oc get deployment istio-ingressgateway -o json | jq '.spec.template.spec.containers[].resources.requests={},.spec.template.spec.containers[].args += ["--proxy-concurrency", "4"]'
+oc get deployment istio-ingressgateway -o json | jq '.spec.template.spec.containers[].resources.requests={},.spec.template.spec.containers[].args += ["--proxy-concurrency", "4"]'

benchmarks/closedmodel-withoutossm.yaml.j2

@@ -0,0 +1,73 @@
+name: closedmodel-withoutossm
+agents:
+{%   if hyperfoil_deployer == "k8s" %}
+  {{ agent }}:
+    image: quay.io/ibm-z-performance/hyperfoil:0.25
+    pod-cpu: "4000m"
+    pod-memory: "4Gi"
+{%   else %}
+  {{ agent }}: {{ hostvars[agent]['ansible_host'] }}:{{ hyperfoil_agent_port }}
+{%   endif %}
+threads: 50
+http:
+{% for index in range(1, (num_apps | int) + 1) %}
+- host: https://app-{{ index }}.{{ wildcard_domain }}
+  allowHttp2: {{ http2 }}
+  sharedConnections: {{ (test_shared_connections | default(100, true)) // (num_apps | int) }}
+{% endfor %}
+phases:
+- warmup:
+    always:
+      isWarmup: true
+      users: {{ test_concurrent_users }}
+      duration: {{ test_rampup_duration }}
+      forks:
+        simple: &simple
+          weight: {{ (fork == "simple" or fork == "all") | ternary(1, 0) }}
+          scenario:
+          - params: &params
+            - set: p <- 1
+            - randomInt:
+                toVar: hostIndex
+                min: 1
+                max: {{ num_apps }}
+          - test:
+            - httpRequest:
+                GET: /mersennePrime?p=${p}
+                authority: app-${hostIndex}.{{ wildcard_domain }}
+                sla:
+                  errorRatio: 0.1
+        proxy: &proxy
+          weight: {{ (fork == "proxy" or fork == "all") | ternary(1, 0) }}
+          scenario:
+          - params: *params
+          - test:
+            - httpRequest:
+                GET: /proxy?p=${p}&url=http://app-${hostIndex}:8080/mersennePrime?p=${p}
+                authority: app-${hostIndex}.{{ wildcard_domain }}
+                sla:
+                  errorRatio: 0.1
+        db: &db
+          weight: {{ (fork == "db" or fork == "all") | ternary(1, 0) }}
+          scenario:
+          - params: *params
+          - test:
+            - randomInt:
+                toVar: size
+                min: 5
+                max: 20
+            - httpRequest:
+                GET: /db?p=${p}&host=app-${hostIndex}&size=${size}
+                authority: app-${hostIndex}.{{ wildcard_domain }}
+                sla:
+                  errorRatio: 0.1
+- steady:
+    always:
+      users: {{ test_concurrent_users }}
+      duration: {{ test_steady_duration }}
+      startAfter: warmup
+      forks:
+        simple: *simple
+        proxy: *proxy
+        db: *db
+

benchmarks/closedmodel.yaml.j2

@@ -1,17 +1,13 @@
 name: closedmodel
 agents:
-  default:
-    extras: -XX:+UseShenandoahGC
-    # -XX:StartFlightRecording=delay=30s,duration=150s,disk=true,filename=/tmp/recording.jfr,settings=profile
-    #stop: false
-{% for agent in groups[hyperfoil_agent_group] %}
 {%   if hyperfoil_deployer == "k8s" %}
   {{ agent }}:
-    node: {{ agent }}
+    image: quay.io/ibm-z-performance/hyperfoil:0.25
+    pod-cpu: "2000m"
+    pod-memory: "2Gi"
 {%   else %}
   {{ agent }}: {{ hostvars[agent]['ansible_host'] }}:{{ hyperfoil_agent_port }}
 {%   endif %}
-{% endfor %}
 threads: 50
 http:
 {% for index in range(1, (num_apps | int) + 1) %}
@@ -54,7 +50,7 @@ phases:
           - test:
             - httpRequest:
                 GET: /mersennePrime?p=${p}
-                authority: app-${hostIndex}.{{ wildcard_domain }}:443
+                authority: app-${hostIndex}.{{ wildcard_domain }}
                 headers:
                   x-variant:
                     fromVar: variant
@@ -75,7 +71,7 @@ phases:
           - test:
             - httpRequest:
                 GET: /proxy?p=${p}&url=http://app-${hostIndex}:8080/mersennePrime?p=${p}
-                authority: app-${hostIndex}.{{ wildcard_domain }}:443
+                authority: app-${hostIndex}.{{ wildcard_domain }}
                 headers:
                   x-variant:
                     fromVar: variant
@@ -103,7 +99,7 @@ phases:
                 max: 20
             - httpRequest:
                 GET: /db?p=${p}&host=app-${hostIndex}&size=${size}
-                authority: app-${hostIndex}.{{ wildcard_domain }}:443
+                authority: app-${hostIndex}.{{ wildcard_domain }}
                 headers:
                   x-variant:
                     fromVar: variant
@@ -128,4 +124,4 @@ phases:
       forks:
         simple: *simple
         proxy: *proxy
-        db: *db
+        db: *db

check-services.sh

@@ -3,10 +3,10 @@
 DOMAIN=$1
 NUM_APPS=$2
 
-REPLICAS=$(oc get deployment -n mesh-control-plane istio-ingressgateway -o json | jq -r .spec.replicas)
+REPLICAS=$(oc get deployment -n istio-system istio-ingressgateway -o json | jq -r .spec.replicas)
 echo "Waiting for all gateways to come up"
 while true; do
-    RUNNING=$(oc get po -n mesh-control-plane -l app=istio-ingressgateway --field-selector 'status.phase=Running' --no-headers | wc -l)
+    RUNNING=$(oc get po -n istio-system -l app=istio-ingressgateway --field-selector 'status.phase=Running' --no-headers | wc -l)
     if [ $RUNNING -eq $REPLICAS ]; then break; fi
     sleep 1;
 done
@@ -28,5 +28,4 @@ if [ $STATUS = "ok" ]; then
 fi
 
 echo "Deleting ingress gateway"
-oc delete po -n mesh-control-plane -l app=istio-ingressgateway
-exit 1;
+exit 1;

clusterrole-fix.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: istio-sidecar-injector-istio-system
+  labels:
+    app: istio-sidecar-injector
+    release: istio
+    istio: sidecar-injector
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["admissionregistration.k8s.io"]
+  resources: ["mutatingwebhookconfigurations"]
+  verbs: ["get", "list", "watch", "patch", "create" ]