From 416812c8c0b386f2e90ceab1b3b00f5840acc6cf Mon Sep 17 00:00:00 2001
From: punda1 <93995817+punda1@users.noreply.github.com>
Date: Tue, 23 Nov 2021 09:39:50 +0000
Subject: [PATCH] US-443997-1: Ability to collect heap dumps, GC logs (#358)

* Changes to specify diagnostic file size from helm chart

* Mount logs volume and Generate heapdump on OOM for hazelcast clustering
service.

* Minor change

* Changes to push logs to cloudwatch

* Removing changes for pushing logs to Cloudwatch

* Changes to introduce diagnostic metric level and file count
properties
---
 .../hazelcast/templates/pega-hz-deployment.yaml       |  5 +++++
 .../templates/pega-hz-environment-config.yaml         |  9 +++++++++
 charts/pega/charts/hazelcast/values.yaml              |  6 +++++-
 terratest/src/test/pega/pega-hz-deployment_test.go    | 11 +++++++----
 4 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/charts/pega/charts/hazelcast/templates/pega-hz-deployment.yaml b/charts/pega/charts/hazelcast/templates/pega-hz-deployment.yaml
index ade5105f2..e58b081a6 100644
--- a/charts/pega/charts/hazelcast/templates/pega-hz-deployment.yaml
+++ b/charts/pega/charts/hazelcast/templates/pega-hz-deployment.yaml
@@ -27,6 +27,8 @@ spec:
         imagePullPolicy: {{ .Values.imagePullPolicy }}
   {{- end }}
         volumeMounts:
+        - name: logs
+          mountPath: "/opt/hazelcast/logs"
         - name: {{ template "pegaVolumeCredentials" }}
           mountPath: "/opt/hazelcast/secrets"
         envFrom:
@@ -58,6 +60,9 @@ spec:
             port: 5701
       restartPolicy: Always
       volumes:
+      # Volume used to mount logs folder
+      - name: logs
+        emptyDir: {}
       # Volume used to mount secret files.
       {{- include "pegaCredentialVolumeTemplate" . | indent 6 }}
       imagePullSecrets:
diff --git a/charts/pega/charts/hazelcast/templates/pega-hz-environment-config.yaml b/charts/pega/charts/hazelcast/templates/pega-hz-environment-config.yaml
index dfeb6cb61..76daed60a 100644
--- a/charts/pega/charts/hazelcast/templates/pega-hz-environment-config.yaml
+++ b/charts/pega/charts/hazelcast/templates/pega-hz-environment-config.yaml
@@ -46,4 +46,13 @@ data:
 {{- if .Values.server.diagnostics_enabled }}
   DIAGNOSTICS_ENABLED: {{ .Values.server.diagnostics_enabled | quote }}
 {{- end }}
+{{- if .Values.server.diagnostics_metric_level }}
+  DIAGNOSTICS_METRIC_LEVEL: {{ .Values.server.diagnostics_metric_level | quote }}
+{{- end }}
+{{- if .Values.server.diagnostic_log_file_size_mb }}
+  DIAGNOSTIC_LOG_FILE_SIZE_MB: {{ .Values.server.diagnostic_log_file_size_mb | quote }}
+{{- end }}
+{{- if .Values.server.diagnostics_file_count }}
+  DIAGNOSTICS_FILE_COUNT: {{ .Values.server.diagnostics_file_count | quote }}
+{{- end }}
 {{ end }}
\ No newline at end of file
diff --git a/charts/pega/charts/hazelcast/values.yaml b/charts/pega/charts/hazelcast/values.yaml
index 6cf013f8c..83a61fc5d 100644
--- a/charts/pega/charts/hazelcast/values.yaml
+++ b/charts/pega/charts/hazelcast/values.yaml
@@ -23,7 +23,8 @@ client:
   clusterName: "PRPC"
 # Server side settings for hazelcast
 server:
-  java_opts: "-Dhazelcast.initial.min.cluster.size=3 -Dhazelcast.jmx=true -XX:MaxRAMPercentage=80.0"
+  java_opts: "-Dhazelcast.initial.min.cluster.size=3 -Dhazelcast.jmx=true
+  -XX:MaxRAMPercentage=80.0 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/hazelcast/logs/heapdump.hprof"
   health_monitoring_level: "OFF"
   operation_generic_thread_count: ""
   operation_thread_count: ""
@@ -36,3 +37,6 @@ server:
   service_dns_timeout: ""
   logging_level: "info"
   diagnostics_enabled: "true"
+  diagnostics_metric_level: "info"
+  diagnostic_log_file_size_mb: "50"
+  diagnostics_file_count: "3"
diff --git a/terratest/src/test/pega/pega-hz-deployment_test.go b/terratest/src/test/pega/pega-hz-deployment_test.go
index 1be79f3f5..ec323bfba 100644
--- a/terratest/src/test/pega/pega-hz-deployment_test.go
+++ b/terratest/src/test/pega/pega-hz-deployment_test.go
@@ -54,10 +54,13 @@ func VerifyHazelcastDeployment(t *testing.T, yamlContent string) {
 			require.Equal(t, intstr.FromInt(5701), statefulsetSpec.Containers[0].ReadinessProbe.HTTPGet.Port)
 			require.Equal(t, "2", statefulsetSpec.Containers[0].Resources.Requests.Cpu().String())
 			require.Equal(t, "2Gi", statefulsetSpec.Containers[0].Resources.Requests.Memory().String())
-			require.Equal(t, statefulsetSpec.Volumes[0].Name, "pega-volume-credentials")
-            require.Equal(t, statefulsetSpec.Volumes[0].Secret.SecretName, "pega-credentials-secret")
-            require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[0].Name, "pega-volume-credentials")
-            require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[0].MountPath, "/opt/hazelcast/secrets")
+			require.Equal(t, statefulsetSpec.Volumes[0].Name, "logs")
+            require.Equal(t, statefulsetSpec.Volumes[1].Name, "pega-volume-credentials")
+            require.Equal(t, statefulsetSpec.Volumes[1].Secret.SecretName, "pega-credentials-secret")
+            require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[0].Name, "logs")
+            require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[0].MountPath, "/opt/hazelcast/logs")
+            require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[1].Name, "pega-volume-credentials")
+            require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[1].MountPath, "/opt/hazelcast/secrets")
 		}
 	}
 }
\ No newline at end of file