add custom exporter into related deployments.

GoogleCloudPlatform · Mar 27, 2024 · 32826d4 · 32826d4
1 parent 16ca73b
commit 32826d4
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 6 deletions.
diff --git a/benchmarks/benchmark/tools/locust-load-inference/build.tf b/benchmarks/benchmark/tools/locust-load-inference/build.tf
@@ -14,3 +14,11 @@ resource "null_resource" "build_and_push_runner_image" {
     command     = "gcloud builds submit --tag ${var.artifact_registry}/locust-runner:latest locust-runner"
   }
 }
+
+resource "null_resource" "build_and_push_exporter_image" {
+
+  provisioner "local-exec" {
+    working_dir = path.module
+    command     = "gcloud builds submit --tag ${var.artifact_registry}/locust-custom-exporter:latest locust-custom-exporter"
+  }
+}
diff --git a/benchmarks/benchmark/tools/locust-load-inference/locust-runner/metrics.yaml b/benchmarks/benchmark/tools/locust-load-inference/locust-runner/metrics.yaml
@@ -9,6 +9,26 @@
             'filter': 'metric.type = "prometheus.googleapis.com/DCGM_FI_DEV_FB_USED/gauge"',
             'type': 'GAUGE',
             'aggregation': ''
+        },
+        'AVG_TEST_TIME': {
+            'filter': 'metric.type = "prometheus/locust_custom_metrics_avg_test_time/gauge"',
+            'type': 'GAUGE',
+            'aggregation': ''
+        },
+        'AVG_OUTPUT_TOKEN_LATENCY': {
+            'filter': 'metric.type = "prometheus/locust_custom_metrics_avg_output_token_latency/gauge"',
+            'type': 'GAUGE',
+            'aggregation': ''
+        },
+        'AVG_TOKENS_RECEIVED': {
+            'filter': 'metric.type = "prometheus/locust_custom_metrics_avg_tokens_received/gauge"',
+            'type': 'GAUGE',
+            'aggregation': ''
+        },
+        'AVG_TOKENS_SENT': {
+            'filter': 'metric.type = "prometheus/locust_custom_metrics_avg_tokens_sent/gauge"',
+            'type': 'GAUGE',
+            'aggregation': ''
         }
     }
 }
diff --git a/...enchmark/tools/locust-load-inference/manifest-templates/locust-master-controller.yaml.tpl b/...enchmark/tools/locust-load-inference/manifest-templates/locust-master-controller.yaml.tpl
@@ -38,12 +38,15 @@ spec:
             - name: loc-master-p2
               containerPort: 5558
               protocol: TCP
+        - name: locust-custom-metrics-exporter
+          image: ${artifact_registry}/locust-custom-exporter:latest
+          ports:
+            - name: cmetrics-port
+              containerPort: 8080
+              protocol: TCP
         - name: locust-metrics-exporter
           image: containersol/locust_exporter
-          env:
-            - name: LOCUST_EXPORTER_WEB_LISTEN_ADDRESS
-              value: ":8080"
           ports:
             - name: metrics-port
-              containerPort: 8080
-              protocol: TCP
+              containerPort: 9646
+              protocol: TCP
diff --git a/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/pod-monitoring.yaml.tpl b/benchmarks/benchmark/tools/locust-load-inference/manifest-templates/pod-monitoring.yaml.tpl
@@ -7,7 +7,8 @@ spec:
   selector:
     matchLabels:
       app: locust-master
-
   endpoints:
   - port: 8080
+    interval: 5s
+  - port: 9646
     interval: 5s