diff --git a/containers/kubernetes/samples/opentelemetry/README.md b/containers/kubernetes/samples/opentelemetry/README.md new file mode 100644 index 0000000000..38ef536529 --- /dev/null +++ b/containers/kubernetes/samples/opentelemetry/README.md @@ -0,0 +1,69 @@ +# README + +## Install OpenTelemetry Collector + +We will be using the OpenTelemetry Operator for Kubernetes to setup OTEL collector. To install the operator in an existing cluster, `cert-manager` is required. + +Use the following commands to install `cert-manager` and the operator: + +``` +# cert-manager +kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml + +# open telemetry operator +kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml +``` + +Once the `opentelemetry-operator` deployment is ready, we need to create an OpenTelemetry Collector instance. + +The `samples/opentelemetry/collector.yaml` configuration provides a good starting point, however, you may change as per your requirement. To apply, use this command: + +``` +kubectl apply -f samples/opentelemetry/collector.yaml +``` + +## Install Jaeger + +We will using the Jaeger Operator for Kubernetes to deploy Jaeger. To install the operator, run: + +``` +kubectl create namespace observability +kubectl create -n observability -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.29.1/jaeger-operator.yaml +``` + +Note that you’ll need to download and customize the Role Bindings if you are using a namespace other than observability. + +Once the `jaeger-operator` deployment in the namespace observability is ready, create a Jaeger instance, like: + +``` +kubectl apply -n observability -f samples/opentelemetry/jaeger.yaml +``` + +## Verify and port forwarding + +Check if the `otel-collector` and `jaeger-query` service has been created: + +``` +kubectl get svc --all-namespaces + +NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +cert-manager cert-manager ClusterIP 10.96.164.235 9402/TCP 9m52s +cert-manager cert-manager-webhook ClusterIP 10.111.113.88 443/TCP 9m52s +observability jaeger-agent ClusterIP None 5775/UDP,5778/TCP,6831/UDP,6832/UDP 99s +observability jaeger-collector ClusterIP 10.102.255.8 9411/TCP,14250/TCP,14267/TCP,14268/TCP 99s +observability jaeger-collector-headless ClusterIP None 9411/TCP,14250/TCP,14267/TCP,14268/TCP 99s +observability jaeger-operator-metrics ClusterIP 10.101.196.203 8443/TCP 6m44s +observability jaeger-query ClusterIP 10.108.235.50 16686/TCP,16685/TCP 99s +opentelemetry-operator-system opentelemetry-operator-controller-manager-metrics-service ClusterIP 10.98.217.60 8443/TCP 9m33s +opentelemetry-operator-system opentelemetry-operator-webhook-service ClusterIP 10.108.3.106 443/TCP 9m33s +opentelemetry-operator-system otel-collector NodePort 10.105.160.200 4317:30080/TCP,8889:31809/TCP 7m39s +< ... other services > +``` + +Now, setup a port forward to the `jaeger-query` service: + +``` +kubectl port-forward service/jaeger-query -n observability 8080:16686 +``` + +You should now be able to access Jaeger at http://localhost:8080/. diff --git a/containers/kubernetes/samples/opentelemetry/collector.yaml b/containers/kubernetes/samples/opentelemetry/collector.yaml new file mode 100644 index 0000000000..b534f59811 --- /dev/null +++ b/containers/kubernetes/samples/opentelemetry/collector.yaml @@ -0,0 +1,132 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-conf + namespace: opentelemetry-operator-system + labels: + app: opentelemetry + component: otel-collector-conf +data: + otel-collector-config: | + receivers: + # Make sure to add the otlp receiver. + # This will open up the receiver on port 4317 + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + processors: + extensions: + health_check: {} + exporters: + jaeger: + endpoint: "jaeger-collector.observability.svc.cluster.local:14250" + insecure: true + prometheus: + endpoint: 0.0.0.0:8889 + namespace: "testapp" + logging: + + service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [jaeger] + + metrics: + receivers: [otlp] + processors: [] + exporters: [prometheus, logging] +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + namespace: opentelemetry-operator-system + labels: + app: opentelemetry + component: otel-collector +spec: + ports: + - name: otlp # Default endpoint for otlp receiver. + port: 4317 + protocol: TCP + targetPort: 4317 + nodePort: 30080 + - name: metrics # Default endpoint for metrics. + port: 8889 + protocol: TCP + targetPort: 8889 + selector: + component: otel-collector + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-collector + namespace: opentelemetry-operator-system + labels: + app: opentelemetry + component: otel-collector +spec: + selector: + matchLabels: + app: opentelemetry + component: otel-collector + minReadySeconds: 5 + progressDeadlineSeconds: 120 + replicas: 1 #TODO - adjust this to your own requirements + template: + metadata: + annotations: + prometheus.io/path: "/metrics" + prometheus.io/port: "8889" + prometheus.io/scrape: "true" + labels: + app: opentelemetry + component: otel-collector + spec: + containers: + - command: + - "/otelcol" + - "--config=/conf/otel-collector-config.yaml" + # Memory Ballast size should be max 1/3 to 1/2 of memory. + - "--mem-ballast-size-mib=683" + env: + - name: GOGC + value: "80" + image: otel/opentelemetry-collector:0.6.0 + name: otel-collector + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 200m + memory: 400Mi + ports: + - containerPort: 4317 # Default endpoint for otlp receiver. + - containerPort: 8889 # Default endpoint for querying metrics. + volumeMounts: + - name: otel-collector-config-vol + mountPath: /conf + # - name: otel-collector-secrets + # mountPath: /secrets + livenessProbe: + httpGet: + path: / + port: 13133 # Health Check extension default port. + readinessProbe: + httpGet: + path: / + port: 13133 # Health Check extension default port. + volumes: + - configMap: + name: otel-collector-conf + items: + - key: otel-collector-config + path: otel-collector-config.yaml + name: otel-collector-config-vol diff --git a/containers/kubernetes/samples/opentelemetry/jaeger.yaml b/containers/kubernetes/samples/opentelemetry/jaeger.yaml new file mode 100644 index 0000000000..c2e48c5351 --- /dev/null +++ b/containers/kubernetes/samples/opentelemetry/jaeger.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: jaegertracing.io/v1 +kind: Jaeger +metadata: + name: jaeger diff --git a/containers/kubernetes/workloads/instance-env/api-catalog-deployment.yaml b/containers/kubernetes/workloads/instance-env/api-catalog-deployment.yaml index 09f0321ef6..20d48f337b 100644 --- a/containers/kubernetes/workloads/instance-env/api-catalog-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/api-catalog-deployment.yaml @@ -85,6 +85,8 @@ spec: fieldPath: metadata.namespace - name: CMMN_LB value: apiml-common-lib/bin/api-layer-lite-lib-all.jar + - name: QUICK_START + value: "-javaagent:/home/zowe/runtime/bin/opentelemetry-javaagent.jar -Dotel.traces.exporter=otlp -Dotel.resource.attributes=service.name=api-catalog -Dotel.exporter.otlp.traces.endpoint=http://34.71.61.249:4317" lifecycle: preStop: exec: @@ -114,7 +116,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/app-server-deployment.yaml b/containers/kubernetes/workloads/instance-env/app-server-deployment.yaml index a62dc42016..425d782d6c 100644 --- a/containers/kubernetes/workloads/instance-env/app-server-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/app-server-deployment.yaml @@ -48,7 +48,7 @@ spec: claimName: zowe-workspace-pvc containers: - name: app-server - image: zowe-docker-release.jfrog.io/ompzowe/app-server:1-ubuntu + image: zowe-docker-snapshot.jfrog.io/ompzowe/app-server:1.28.0-ubuntu.users-jack-trace imagePullPolicy: Always resources: requests: @@ -108,7 +108,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/caching-deployment.yaml b/containers/kubernetes/workloads/instance-env/caching-deployment.yaml index 283b4404f8..e98b71f1da 100644 --- a/containers/kubernetes/workloads/instance-env/caching-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/caching-deployment.yaml @@ -85,6 +85,8 @@ spec: fieldPath: metadata.namespace - name: CMMN_LB value: apiml-common-lib/bin/api-layer-lite-lib-all.jar + - name: QUICK_START + value: "-javaagent:/home/zowe/runtime/bin/opentelemetry-javaagent.jar -Dotel.traces.exporter=otlp -Dotel.resource.attributes=service.name=caching-service -Dotel.exporter.otlp.traces.endpoint=http://34.71.61.249:4317" lifecycle: preStop: exec: @@ -114,7 +116,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/cleanup-static-definitions-cronjob.yaml b/containers/kubernetes/workloads/instance-env/cleanup-static-definitions-cronjob.yaml index d470f8f00e..0871757fbf 100644 --- a/containers/kubernetes/workloads/instance-env/cleanup-static-definitions-cronjob.yaml +++ b/containers/kubernetes/workloads/instance-env/cleanup-static-definitions-cronjob.yaml @@ -45,7 +45,7 @@ spec: containers: - name: cleanup-static-definitions # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/discovery-statefulset.yaml b/containers/kubernetes/workloads/instance-env/discovery-statefulset.yaml index 6e0fe60fc6..7781f5a100 100644 --- a/containers/kubernetes/workloads/instance-env/discovery-statefulset.yaml +++ b/containers/kubernetes/workloads/instance-env/discovery-statefulset.yaml @@ -88,6 +88,8 @@ spec: fieldPath: metadata.namespace - name: CMMN_LB value: apiml-common-lib/bin/api-layer-lite-lib-all.jar + - name: QUICK_START + value: "-javaagent:/home/zowe/runtime/bin/opentelemetry-javaagent.jar -Dotel.traces.exporter=otlp -Dotel.resource.attributes=service.name=discovery -Dotel.exporter.otlp.traces.endpoint=http://34.71.61.249:4317" lifecycle: preStop: exec: @@ -117,7 +119,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/explorer-ip-job.yaml b/containers/kubernetes/workloads/instance-env/explorer-ip-job.yaml index 6a5364b660..54d8332d4f 100644 --- a/containers/kubernetes/workloads/instance-env/explorer-ip-job.yaml +++ b/containers/kubernetes/workloads/instance-env/explorer-ip-job.yaml @@ -78,7 +78,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/explorer-jes-deployment.yaml b/containers/kubernetes/workloads/instance-env/explorer-jes-deployment.yaml index ff07cc2a04..4d4febd5b5 100644 --- a/containers/kubernetes/workloads/instance-env/explorer-jes-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/explorer-jes-deployment.yaml @@ -108,7 +108,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/explorer-mvs-deployment.yaml b/containers/kubernetes/workloads/instance-env/explorer-mvs-deployment.yaml index 6e4488b518..07dd00e754 100644 --- a/containers/kubernetes/workloads/instance-env/explorer-mvs-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/explorer-mvs-deployment.yaml @@ -108,7 +108,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/explorer-uss-deployment.yaml b/containers/kubernetes/workloads/instance-env/explorer-uss-deployment.yaml index 7663034ee5..5037972a71 100644 --- a/containers/kubernetes/workloads/instance-env/explorer-uss-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/explorer-uss-deployment.yaml @@ -108,7 +108,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/files-api-deployment.yaml b/containers/kubernetes/workloads/instance-env/files-api-deployment.yaml index 0b80bcb553..fbc36dfe16 100644 --- a/containers/kubernetes/workloads/instance-env/files-api-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/files-api-deployment.yaml @@ -48,7 +48,7 @@ spec: claimName: zowe-workspace-pvc containers: - name: files-api - image: zowe-docker-release.jfrog.io/ompzowe/files-api:1-ubuntu + image: zowe-docker-snapshot.jfrog.io/ompzowe/files-api:1.0.17-SNAPSHOT-ubuntu.users-jack-trace imagePullPolicy: Always resources: requests: @@ -108,7 +108,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/gateway-deployment.yaml b/containers/kubernetes/workloads/instance-env/gateway-deployment.yaml index 44fe1ab83d..0c4c48b497 100644 --- a/containers/kubernetes/workloads/instance-env/gateway-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/gateway-deployment.yaml @@ -85,6 +85,8 @@ spec: fieldPath: metadata.namespace - name: CMMN_LB value: apiml-common-lib/bin/api-layer-lite-lib-all.jar + - name: QUICK_START + value: "-javaagent:/home/zowe/runtime/bin/opentelemetry-javaagent.jar -Dotel.traces.exporter=otlp -Dotel.resource.attributes=service.name=gateway -Dotel.exporter.otlp.traces.endpoint=http://34.71.61.249:4317" lifecycle: preStop: exec: @@ -114,7 +116,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/kubernetes/workloads/instance-env/jobs-api-deployment.yaml b/containers/kubernetes/workloads/instance-env/jobs-api-deployment.yaml index 8ba890fd27..b5189fafff 100644 --- a/containers/kubernetes/workloads/instance-env/jobs-api-deployment.yaml +++ b/containers/kubernetes/workloads/instance-env/jobs-api-deployment.yaml @@ -48,7 +48,7 @@ spec: claimName: zowe-workspace-pvc containers: - name: jobs-api - image: zowe-docker-release.jfrog.io/ompzowe/jobs-api:1-ubuntu + image: zowe-docker-snapshot.jfrog.io/ompzowe/jobs-api:1.0.15-SNAPSHOT-ubuntu.users-jack-trace imagePullPolicy: Always resources: requests: @@ -108,7 +108,7 @@ spec: initContainers: - name: init-zowe # image: zowe-docker-release.jfrog.io/ompzowe/zowe-launch-scripts:latest - image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.25.0-ubuntu.staging + image: zowe-docker-snapshot.jfrog.io/ompzowe/zowe-launch-scripts:1.28.0-ubuntu.users-jack-test-zlux-tracing imagePullPolicy: Always resources: requests: diff --git a/containers/zowe-launch-scripts/prepare.sh b/containers/zowe-launch-scripts/prepare.sh index be98437256..58e85a8ff8 100755 --- a/containers/zowe-launch-scripts/prepare.sh +++ b/containers/zowe-launch-scripts/prepare.sh @@ -226,6 +226,11 @@ rm -fr zlux-app-manager zlux-build zlux-platform cd "${REPO_ROOT_DIR}" rm -f zlux-core.tar +############################### +# add otel java agent +echo ">>>>> copy opentelemetry-javaagent.jar" +curl --location https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.12.0/opentelemetry-javaagent.jar --output "${BASE_DIR}/${WORK_DIR}/bin/opentelemetry-javaagent.jar" + ############################### # copy to target context echo ">>>>> copy to target build context" diff --git a/manifest.json.template b/manifest.json.template index 4e5503c9e2..a7576720a5 100644 --- a/manifest.json.template +++ b/manifest.json.template @@ -12,8 +12,9 @@ }, "binaryDependencies": { "org.zowe.zlux.zlux-core": { - "version": "~1.28.0-v1.x-staging", - "artifact": "*.pax" + "repository": "libs-snapshot-local", + "version": "1.28.0-114-v1.x-staging-zlux-build", + "artifact": "zlux-core-1.28.0-20220512.145208.pax" }, "org.zowe.zlux.zss-auth": { "version": "~1.28.0-V1.X-STAGING",