Decrease scrape interval of metrics from TGI and DCGM to 15s in bench…

…mark (#772)

Decrease scrape interval of metrics from TGI and DCGM from 30s to 15s. To facilitate this, also decrease DCGM's interval to 5s to ensure fresh metrics.

benchmarks/inference-server/text-generation-inference/hpa-templates/dcgm-podmonitoring.yaml.tftpl

@@ -26,7 +26,7 @@ spec:
       app: nvidia-dcgm-exporter
   endpoints:
     - port: metrics
-      interval: 30s
+      interval: 15s
       metricRelabeling:
         # Change the DCGM metric name that we want to use in HPA to lowercase.
         # This is because HPA doesn't work with uppercase external metrics:

benchmarks/inference-server/text-generation-inference/monitoring-templates/tgi-podmonitoring.yaml.tftpl

@@ -9,4 +9,4 @@ spec:
       app: tgi
   endpoints:
   - port: 80
-    interval: 20s
+    interval: 15s

benchmarks/infra/stage-2/modules/gke-setup/modules/nvidia-dcgm/manifest-templates/02-ds-exporter.yaml

@@ -54,7 +54,7 @@ spec:
           image: nvcr.io/nvidia/k8s/dcgm-exporter:3.1.8-3.1.5-ubuntu20.04
           command: ["/bin/bash", "-c"]
           args:
-            - hostname $NODE_NAME; dcgm-exporter --remote-hostengine-info $(NODE_IP) --collectors /etc/dcgm-exporter/counters.csv
+            - hostname $NODE_NAME; dcgm-exporter --remote-hostengine-info $(NODE_IP) --collect-interval 5000 --collectors /etc/dcgm-exporter/counters.csv
           ports:
             - name: metrics
               containerPort: 9400