Update counter names to avoid clashes with project_id, ad k8s configs

cloudspannerecosystem · Feb 5, 2024 · 21983f9 · 21983f9
1 parent c541b19
commit 21983f9
Show file tree

Hide file tree

Showing 16 changed files with 503 additions and 41 deletions.
diff --git a/kubernetes/decoupled/autoscaler-pkg/networkpolicy.yaml b/kubernetes/decoupled/autoscaler-pkg/networkpolicy.yaml
@@ -40,3 +40,23 @@ spec:
       ports:
         - protocol: TCP
           port: 3000
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-otel-submitter-to-collector
+  namespace: spanner-autoscaler # kpt-set: ${namespace}
+spec:
+  podSelector:
+    matchLabels:
+      app: otel-collector
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              otel-submitter: "true"
+      ports:
+        - protocol: TCP
+          port: 4317
diff --git a/kubernetes/decoupled/autoscaler-pkg/otel-collector/Kptfile b/kubernetes/decoupled/autoscaler-pkg/otel-collector/Kptfile
@@ -0,0 +1,21 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: kpt.dev/v1
+kind: Kptfile
+metadata:
+  name: otel-collector
+  annotations:
+    config.kubernetes.io/local-config: "true"
+info:
+  description: Config for OpenTelemetry Collector component of Spanner autoscaler
diff --git a/kubernetes/decoupled/autoscaler-pkg/otel-collector/otel-collector.yaml b/kubernetes/decoupled/autoscaler-pkg/otel-collector/otel-collector.yaml
@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: otel-collector
+  namespace: spanner-autoscaler # kpt-set: ${namespace}
+  labels:
+    app: otel-collector
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: otel-collector
+  template:
+    metadata:
+      labels:
+        app: otel-collector
+    spec:
+      containers:
+      - name: otel-collector
+        image: otel/opentelemetry-collector-contrib:0.93.0
+        args:
+        - --config
+        - /etc/otel/config.yaml
+        volumeMounts:
+        - mountPath: /etc/otel/
+          name: otel-config
+      volumes:
+      - name: otel-config
+        configMap:
+          name: otel-config
+      nodeSelector:
+        iam.gke.io/gke-metadata-server-enabled: "true"
+      serviceAccountName: otel-collector-sa
+      automountServiceAccountToken: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  namespace: spanner-autoscaler # kpt-set: ${namespace}
+spec:
+  type: ClusterIP
+  selector:
+    app: otel-collector
+  ports:
+    - protocol: TCP
+      port: 4317
+      targetPort: 4317
diff --git a/kubernetes/decoupled/autoscaler-pkg/poller/poller-hourly.yaml b/kubernetes/decoupled/autoscaler-pkg/poller/poller-hourly.yaml
@@ -25,6 +25,7 @@ spec:
         metadata:
           labels:
             app: poller
+            otel-submitter: "true"
         spec:
           containers:
           - name: poller
@@ -38,6 +39,12 @@ spec:
             env:
             - name: AUTOSCALER_CONFIG
               value: "/etc/autoscaler-config/autoscaler-config-hourly.yaml"
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OLTP_COLLECTOR_URL
+              value: "http://otel-collector:4713/"
             volumeMounts:
             - name: config-volume
               mountPath: /etc/autoscaler-config

diff --git a/kubernetes/decoupled/autoscaler-pkg/poller/poller.yaml b/kubernetes/decoupled/autoscaler-pkg/poller/poller.yaml
@@ -25,6 +25,7 @@ spec:
         metadata:
           labels:
             app: poller
+            otel-submitter: "true"
         spec:
           containers:
           - name: poller
@@ -35,6 +36,13 @@ spec:
                 cpu: "250m"
               limits:
                 memory: "256Mi"
+            env:
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OLTP_COLLECTOR_URL
+              value: "http://otel-collector:4713/"
             volumeMounts:
             - name: config-volume
               mountPath: /etc/autoscaler-config

diff --git a/kubernetes/decoupled/autoscaler-pkg/scaler/scaler.yaml b/kubernetes/decoupled/autoscaler-pkg/scaler/scaler.yaml
@@ -28,6 +28,7 @@ spec:
     metadata:
       labels:
         app: scaler
+        otel-submitter: "true"
     spec:
       containers:
       - name: scaler
@@ -40,6 +41,13 @@ spec:
             cpu: "100m"
           limits:
             memory: "256Mi"
+        env:
+          - name: K8S_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: OLTP_COLLECTOR_URL
+            value: "http://otel-collector:4713/"
       nodeSelector:
         iam.gke.io/gke-metadata-server-enabled: "true"
       serviceAccountName: scaler-sa

diff --git a/kubernetes/unified/autoscaler-pkg/networkpolicy.yaml b/kubernetes/unified/autoscaler-pkg/networkpolicy.yaml
@@ -21,3 +21,23 @@ spec:
   podSelector: {}
   policyTypes:
   - Ingress
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-otel-submitter-to-collector
+  namespace: spanner-autoscaler # kpt-set: ${namespace}
+spec:
+  podSelector:
+    matchLabels:
+      app: otel-collector
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              otel-submitter: "true"
+      ports:
+        - protocol: TCP
+          port: 4317
diff --git a/kubernetes/unified/autoscaler-pkg/otel-collector/Kptfile b/kubernetes/unified/autoscaler-pkg/otel-collector/Kptfile
@@ -0,0 +1,21 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: kpt.dev/v1
+kind: Kptfile
+metadata:
+  name: otel-collector
+  annotations:
+    config.kubernetes.io/local-config: "true"
+info:
+  description: Config for OpenTelemetry Collector component of Spanner autoscaler
diff --git a/kubernetes/unified/autoscaler-pkg/otel-collector/otel-collector.yaml b/kubernetes/unified/autoscaler-pkg/otel-collector/otel-collector.yaml
@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: otel-collector
+  namespace: spanner-autoscaler # kpt-set: ${namespace}
+  labels:
+    app: otel-collector
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: otel-collector
+  template:
+    metadata:
+      labels:
+        app: otel-collector
+    spec:
+      containers:
+      - name: otel-collector
+        image: otel/opentelemetry-collector-contrib:0.93.0
+        args:
+        - --config
+        - /etc/otel/config.yaml
+        volumeMounts:
+        - mountPath: /etc/otel/
+          name: otel-config
+      volumes:
+      - name: otel-config
+        configMap:
+          name: otel-config
+      nodeSelector:
+        iam.gke.io/gke-metadata-server-enabled: "true"
+      serviceAccountName: otel-collector-sa
+      automountServiceAccountToken: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  namespace: spanner-autoscaler # kpt-set: ${namespace}
+spec:
+  type: ClusterIP
+  selector:
+    app: otel-collector
+  ports:
+    - protocol: TCP
+      port: 4317
+      targetPort: 4317
diff --git a/kubernetes/unified/autoscaler-pkg/scaler/scaler-hourly.yaml b/kubernetes/unified/autoscaler-pkg/scaler/scaler-hourly.yaml
@@ -25,6 +25,7 @@ spec:
         metadata:
           labels:
             app: scaler
+            otel-submitter: "true"
         spec:
           containers:
           - name: scaler
@@ -38,6 +39,12 @@ spec:
             env:
             - name: AUTOSCALER_CONFIG
               value: "/etc/autoscaler-config/autoscaler-config-hourly.yaml"
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OLTP_COLLECTOR_URL
+              value: "http://otel-collector:4713/"
             volumeMounts:
             - name: config-volume
               mountPath: /etc/autoscaler-config

diff --git a/kubernetes/unified/autoscaler-pkg/scaler/scaler.yaml b/kubernetes/unified/autoscaler-pkg/scaler/scaler.yaml
@@ -25,6 +25,7 @@ spec:
         metadata:
           labels:
             app: scaler
+            otel-submitter: "true"
         spec:
           containers:
           - name: scaler
@@ -35,6 +36,13 @@ spec:
                 cpu: "250m"
               limits:
                 memory: "256Mi"
+            env:
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OLTP_COLLECTOR_URL
+              value: "http://otel-collector:4713/"
             volumeMounts:
             - name: config-volume
               mountPath: /etc/autoscaler-config

diff --git a/src/autoscaler-common/counters_base.js b/src/autoscaler-common/counters_base.js
@@ -25,7 +25,10 @@ const {MeterProvider, PeriodicExportingMetricReader} =
 const {Resource} = require('@opentelemetry/resources');
 const {MetricExporter: GcpMetricExporter} =
   require('@google-cloud/opentelemetry-cloud-monitoring-exporter');
-const {GcpDetectorSync} = require('@google-cloud/opentelemetry-resource-util');
+const {OTLPMetricExporter} =
+  require('@opentelemetry/exporter-metrics-otlp-grpc');
+const {GcpDetectorSync} =
+   require('@google-cloud/opentelemetry-resource-util');
 const {SemanticResourceAttributes} =
   require('@opentelemetry/semantic-conventions');
 const OpenTelemetryApi = require('@opentelemetry/api');
@@ -53,6 +56,11 @@ const AUTOSCALER_RESOURCE_ATTRIBUTES = {
   [SemanticResourceAttributes.SERVICE_VERSION]: '1.0',
 };
 
+const COUNTER_ATTRIBUTE_NAMES = {
+  SPANNER_PROJECT_ID: 'spanner_project_id',
+  SPANNER_INSTANCE_ID: 'spanner_project_id',
+};
+
 /**
  * The prefix to use for any autoscaler counters.
  */
@@ -179,19 +187,39 @@ async function initMetrics() {
     await gcpResources.waitForAsyncAttributes();
     logger.debug('got GCP resources %o', gcpResources);
 
+
+    if (process.env.KUBERNETES_SERVICE_HOST) {
+      if (process.env.K8S_POD_NAME) {
+        gcpResources[SemanticResourceAttributes.K8S_POD_NAME] =
+            process.env.K8S_POD_NAME;
+      } else {
+        logger.warn('WARNING: running under Kubernetes, but K8S_POD_NAME ' +
+        'environment variable is not set. ' +
+        'This may lead to duplicate TimeSeries errors');
+      }
+    }
+
+    let exporter;
+    if (process.env.OLTP_COLLECTOR_URL) {
+      logger.info(`Counters using OLTP Metrics exporter to ${
+        process.env.OLTP_COLLECTOR_URL}`);
+      exporter = new OTLPMetricExporter({url: process.env.OLTP_COLLECTOR_URL});
+    } else {
+      logger.info('Counters exporting directly to GCP monitoring');
+      exporter = new GcpMetricExporter();
+    }
+
     meterProvider = new MeterProvider({
       resource: new Resource(AUTOSCALER_RESOURCE_ATTRIBUTES)
           .merge(gcpResources),
+      readers: [
+        new PeriodicExportingMetricReader({
+          exportIntervalMillis: PERIODIC_FLUSH_INTERVAL,
+          exportTimeoutMillis: PERIODIC_FLUSH_INTERVAL,
+          exporter: exporter,
+        }),
+      ],
     });
-
-    // Export metrics to GCP.
-    const gcpMetricExporter = new GcpMetricExporter();
-
-    meterProvider.addMetricReader(new PeriodicExportingMetricReader({
-      exportIntervalMillis: PERIODIC_FLUSH_INTERVAL,
-      exportTimeoutMillis: PERIODIC_FLUSH_INTERVAL,
-      exporter: gcpMetricExporter,
-    }));
   } catch (e) {
     // report failures to other waiters.
     rejectPendingInit(e);
@@ -346,6 +374,7 @@ function setTryFlushEnabled(newTryFlushEnabled) {
 }
 
 module.exports = {
+  COUNTER_ATTRIBUTE_NAMES,
   createCounters,
   incCounter,
   tryFlush,