zowe · jackjia-ibm · May 12, 2022 · May 12, 2022 · May 12, 2022 · May 12, 2022
diff --git a/containers/kubernetes/samples/opentelemetry/README.md b/containers/kubernetes/samples/opentelemetry/README.md
@@ -0,0 +1,69 @@
+# README
+
+## Install OpenTelemetry Collector
+
+We will be using the OpenTelemetry Operator for Kubernetes to setup OTEL collector. To install the operator in an existing cluster, `cert-manager` is required.
+
+Use the following commands to install `cert-manager` and the operator:
+
+```
+# cert-manager
+kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml
+
+# open telemetry operator
+kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml
+```
+
+Once the `opentelemetry-operator` deployment is ready, we need to create an OpenTelemetry Collector instance.
+
+The `samples/opentelemetry/collector.yaml` configuration provides a good starting point, however, you may change as per your requirement. To apply, use this command:
+
+```
+kubectl apply -f samples/opentelemetry/collector.yaml
+```
+
+## Install Jaeger
+
+We will using the Jaeger Operator for Kubernetes to deploy Jaeger. To install the operator, run:
+
+```
+kubectl create namespace observability
+kubectl create -n observability -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.29.1/jaeger-operator.yaml
+```
+
+Note that you’ll need to download and customize the Role Bindings if you are using a namespace other than observability.
+
+Once the `jaeger-operator` deployment in the namespace observability is ready, create a Jaeger instance, like:
+
+```
+kubectl apply -n observability -f samples/opentelemetry/jaeger.yaml
+```
+
+## Verify and port forwarding
+
+Check if the `otel-collector` and `jaeger-query` service has been created:
+
+```
+kubectl get svc --all-namespaces
+
+NAMESPACE                       NAME                                                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                  AGE
+cert-manager                    cert-manager                                                ClusterIP   10.96.164.235    <none>        9402/TCP                                 9m52s
+cert-manager                    cert-manager-webhook                                        ClusterIP   10.111.113.88    <none>        443/TCP                                  9m52s
+observability                   jaeger-agent                                                ClusterIP   None             <none>        5775/UDP,5778/TCP,6831/UDP,6832/UDP      99s
+observability                   jaeger-collector                                            ClusterIP   10.102.255.8     <none>        9411/TCP,14250/TCP,14267/TCP,14268/TCP   99s
+observability                   jaeger-collector-headless                                   ClusterIP   None             <none>        9411/TCP,14250/TCP,14267/TCP,14268/TCP   99s
+observability                   jaeger-operator-metrics                                     ClusterIP   10.101.196.203   <none>        8443/TCP                                 6m44s
+observability                   jaeger-query                                                ClusterIP   10.108.235.50    <none>        16686/TCP,16685/TCP                      99s
+opentelemetry-operator-system   opentelemetry-operator-controller-manager-metrics-service   ClusterIP   10.98.217.60     <none>        8443/TCP                                 9m33s
+opentelemetry-operator-system   opentelemetry-operator-webhook-service                      ClusterIP   10.108.3.106     <none>        443/TCP                                  9m33s
+opentelemetry-operator-system   otel-collector                                              NodePort    10.105.160.200   <none>        4317:30080/TCP,8889:31809/TCP            7m39s
+< ... other services >
+```
+
+Now, setup a port forward to the `jaeger-query` service:
+
+```
+kubectl port-forward service/jaeger-query -n observability 8080:16686
+```
+
+You should now be able to access Jaeger at http://localhost:8080/.
diff --git a/containers/kubernetes/samples/opentelemetry/collector.yaml b/containers/kubernetes/samples/opentelemetry/collector.yaml
@@ -0,0 +1,132 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-conf
+  namespace: opentelemetry-operator-system
+  labels:
+    app: opentelemetry
+    component: otel-collector-conf
+data:
+  otel-collector-config: |
+    receivers:
+      # Make sure to add the otlp receiver.
+      # This will open up the receiver on port 4317
+      otlp:
+        protocols:
+          grpc:
+            endpoint: "0.0.0.0:4317"
+    processors:
+    extensions:
+      health_check: {}
+    exporters:
+      jaeger:
+        endpoint: "jaeger-collector.observability.svc.cluster.local:14250"
+        insecure: true
+      prometheus:
+        endpoint: 0.0.0.0:8889
+        namespace: "testapp"
+      logging:
+
+    service:
+      extensions: [health_check]
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: []
+          exporters: [jaeger]
+
+        metrics:
+          receivers: [otlp]
+          processors: []
+          exporters: [prometheus, logging]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  namespace: opentelemetry-operator-system
+  labels:
+    app: opentelemetry
+    component: otel-collector
+spec:
+  ports:
+    - name: otlp # Default endpoint for otlp receiver.
+      port: 4317
+      protocol: TCP
+      targetPort: 4317
+      nodePort: 30080
+    - name: metrics # Default endpoint for metrics.
+      port: 8889
+      protocol: TCP
+      targetPort: 8889
+  selector:
+    component: otel-collector
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: otel-collector
+  namespace: opentelemetry-operator-system
+  labels:
+    app: opentelemetry
+    component: otel-collector
+spec:
+  selector:
+    matchLabels:
+      app: opentelemetry
+      component: otel-collector
+  minReadySeconds: 5
+  progressDeadlineSeconds: 120
+  replicas: 1 #TODO - adjust this to your own requirements
+  template:
+    metadata:
+      annotations:
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "8889"
+        prometheus.io/scrape: "true"
+      labels:
+        app: opentelemetry
+        component: otel-collector
+    spec:
+      containers:
+        - command:
+            - "/otelcol"
+            - "--config=/conf/otel-collector-config.yaml"
+            # Memory Ballast size should be max 1/3 to 1/2 of memory.
+            - "--mem-ballast-size-mib=683"
+          env:
+            - name: GOGC
+              value: "80"
+          image: otel/opentelemetry-collector:0.6.0
+          name: otel-collector
+          resources:
+            limits:
+              cpu: 1
+              memory: 2Gi
+            requests:
+              cpu: 200m
+              memory: 400Mi
+          ports:
+            - containerPort: 4317 # Default endpoint for otlp receiver.
+            - containerPort: 8889 # Default endpoint for querying metrics.
+          volumeMounts:
+            - name: otel-collector-config-vol
+              mountPath: /conf
+          # - name: otel-collector-secrets
+          #   mountPath: /secrets
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 13133 # Health Check extension default port.
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 13133 # Health Check extension default port.
+      volumes:
+        - configMap:
+            name: otel-collector-conf
+            items:
+              - key: otel-collector-config
+                path: otel-collector-config.yaml
+          name: otel-collector-config-vol
diff --git a/containers/kubernetes/samples/opentelemetry/jaeger.yaml b/containers/kubernetes/samples/opentelemetry/jaeger.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: jaegertracing.io/v1
+kind: Jaeger
+metadata:
+  name: jaeger
diff --git a/containers/kubernetes/workloads/instance-env/api-catalog-deployment.yaml b/containers/kubernetes/workloads/instance-env/api-catalog-deployment.yaml
@@ -85,6 +85,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: CMMN_LB
               value: apiml-common-lib/bin/api-layer-lite-lib-all.jar
+            - name: QUICK_START
+              value: "-javaagent:/home/zowe/runtime/bin/opentelemetry-javaagent.jar -Dotel.traces.exporter=otlp -Dotel.resource.attributes=service.name=api-catalog -Dotel.exporter.otlp.traces.endpoint=http://34.71.61.249:4317"
           lifecycle:
             preStop:
               exec:
@@ -114,7 +116,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/app-server-deployment.yaml b/containers/kubernetes/workloads/instance-env/app-server-deployment.yaml
@@ -48,7 +48,7 @@ spec:
             claimName: zowe-workspace-pvc
       containers:
         - name: app-server
-          image: zowe-docker-release.jfrog.io/ompzowe/app-server:1-ubuntu
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/app-server:1.28.0-ubuntu.users-jack-trace
           imagePullPolicy: Always
           resources:
             requests:
@@ -108,7 +108,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/caching-deployment.yaml b/containers/kubernetes/workloads/instance-env/caching-deployment.yaml
@@ -85,6 +85,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: CMMN_LB
               value: apiml-common-lib/bin/api-layer-lite-lib-all.jar
+            - name: QUICK_START
+              value: "-javaagent:/home/zowe/runtime/bin/opentelemetry-javaagent.jar -Dotel.traces.exporter=otlp -Dotel.resource.attributes=service.name=caching-service -Dotel.exporter.otlp.traces.endpoint=http://34.71.61.249:4317"
           lifecycle:
             preStop:
               exec:
@@ -114,7 +116,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/cleanup-static-definitions-cronjob.yaml b/containers/kubernetes/workloads/instance-env/cleanup-static-definitions-cronjob.yaml
@@ -45,7 +45,7 @@ spec:
           containers:
             - name: cleanup-static-definitions
               # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-              image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+              image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
               imagePullPolicy: Always
               resources:
                 requests:

diff --git a/containers/kubernetes/workloads/instance-env/discovery-statefulset.yaml b/containers/kubernetes/workloads/instance-env/discovery-statefulset.yaml
@@ -88,6 +88,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: CMMN_LB
               value: apiml-common-lib/bin/api-layer-lite-lib-all.jar
+            - name: QUICK_START
+              value: "-javaagent:/home/zowe/runtime/bin/opentelemetry-javaagent.jar -Dotel.traces.exporter=otlp -Dotel.resource.attributes=service.name=discovery -Dotel.exporter.otlp.traces.endpoint=http://34.71.61.249:4317"
           lifecycle:
             preStop:
               exec:
@@ -117,7 +119,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/explorer-ip-job.yaml b/containers/kubernetes/workloads/instance-env/explorer-ip-job.yaml
@@ -78,7 +78,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/explorer-jes-deployment.yaml b/containers/kubernetes/workloads/instance-env/explorer-jes-deployment.yaml
@@ -108,7 +108,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/explorer-mvs-deployment.yaml b/containers/kubernetes/workloads/instance-env/explorer-mvs-deployment.yaml
@@ -108,7 +108,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/explorer-uss-deployment.yaml b/containers/kubernetes/workloads/instance-env/explorer-uss-deployment.yaml
@@ -108,7 +108,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/files-api-deployment.yaml b/containers/kubernetes/workloads/instance-env/files-api-deployment.yaml
@@ -48,7 +48,7 @@ spec:
             claimName: zowe-workspace-pvc
       containers:
         - name: files-api
-          image: zowe-docker-release.jfrog.io/ompzowe/files-api:1-ubuntu
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/files-api:1.0.17-SNAPSHOT-ubuntu.users-jack-trace
           imagePullPolicy: Always
           resources:
             requests:
@@ -108,7 +108,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/gateway-deployment.yaml b/containers/kubernetes/workloads/instance-env/gateway-deployment.yaml
@@ -85,6 +85,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: CMMN_LB
               value: apiml-common-lib/bin/api-layer-lite-lib-all.jar
+            - name: QUICK_START
+              value: "-javaagent:/home/zowe/runtime/bin/opentelemetry-javaagent.jar -Dotel.traces.exporter=otlp -Dotel.resource.attributes=service.name=gateway -Dotel.exporter.otlp.traces.endpoint=http://34.71.61.249:4317"
           lifecycle:
             preStop:
               exec:
@@ -114,7 +116,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/kubernetes/workloads/instance-env/jobs-api-deployment.yaml b/containers/kubernetes/workloads/instance-env/jobs-api-deployment.yaml
@@ -48,7 +48,7 @@ spec:
             claimName: zowe-workspace-pvc
       containers:
         - name: jobs-api
-          image: zowe-docker-release.jfrog.io/ompzowe/jobs-api:1-ubuntu
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/jobs-api:1.0.15-SNAPSHOT-ubuntu.users-jack-trace
           imagePullPolicy: Always
           resources:
             requests:
@@ -108,7 +108,7 @@ spec:
       initContainers:
         - name: init-zowe
           # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest
-          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging
+          image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing
           imagePullPolicy: Always
           resources:
             requests:

diff --git a/containers/zowe-launch-scripts/prepare.sh b/containers/zowe-launch-scripts/prepare.sh
@@ -226,6 +226,11 @@ rm -fr zlux-app-manager zlux-build zlux-platform
 cd "${REPO_ROOT_DIR}"
 rm -f zlux-core.tar
 
+###############################
+# add otel java agent
+echo ">>>>> copy opentelemetry-javaagent.jar"
+curl --location https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.12.0/opentelemetry-javaagent.jar --output "${BASE_DIR}/${WORK_DIR}/bin/opentelemetry-javaagent.jar"
+
 ###############################
 # copy to target context
 echo ">>>>> copy to target build context"