Support liveness in backend config in branch 0.21 (#772)

* Add liveness in backend config (#767) * Add liveness config in backendconfig * Add ci * Add pod field * Fix the error that backendconfig doesn't work when no env specified (#770) * Fix the error that backendconfig doesn't work when no env specified * fix ci * fix ci * Install gcompat in java runner images (#761) * Install gcompat in java runner images * Bump go to 1.22.4 * Fix oauth2 * Fix oauth2 settings * Disable legacy oauth2 params test case * Add testcase for google pubsub io * Set gsa credential * Set GSA credential * fix ci * print expected result
streamnative · Jul 12, 2024 · 43a7e84 · 43a7e84
1 parent b7d924a
commit 43a7e84
Show file tree

Hide file tree

Showing 61 changed files with 3,310 additions and 205 deletions.
diff --git a/.ci/clusters/global_backend_config.yaml b/.ci/clusters/global_backend_config.yaml
@@ -3,6 +3,11 @@ kind: BackendConfig
 metadata:
   name: global-backend-config
 spec:
+  autoUpdate: true
   env:
     global1: globalvalue1
-    shared1: fromglobal
+    shared1: fromglobal
+  pod:
+    liveness:
+      initialDelaySeconds: 10
+      periodSeconds: 30
diff --git a/.ci/clusters/global_backend_config_without_env.yaml b/.ci/clusters/global_backend_config_without_env.yaml
@@ -0,0 +1,10 @@
+apiVersion: compute.functionmesh.io/v1alpha1
+kind: BackendConfig
+metadata:
+  name: global-backend-config
+spec:
+  autoUpdate: true
+  pod:
+    liveness:
+      initialDelaySeconds: 10
+      periodSeconds: 30
diff --git a/.ci/clusters/values_skywalking_e2e_cluster_with_oauth.yaml b/.ci/clusters/values_skywalking_e2e_cluster_with_oauth.yaml
@@ -135,12 +135,12 @@ auth:
     enabled: true
     provider: "oauth2"
     oauth2:
-      issuerUrl: https://sts.windows.net/06a8a086-ae6e-45b5-a22e-ad90de23013e/
-      issuerUrlParam: https://sts.windows.net/06a8a086-ae6e-45b5-a22e-ad90de23013e/v2.0
-      audience: api://56c1bd14-3ba7-4804-b47b-d46de6dce33e
-      audienceParam: api://56c1bd14-3ba7-4804-b47b-d46de6dce33e/.default
-      brokerClientCredential: '{"client_id":"CLIENT_ID","client_secret":"CLIENT_SECRET","issuer_url":"https://sts.windows.net/06a8a086-ae6e-45b5-a22e-ad90de23013e/v2.0"}'
-      subjectClaim: appid
-      adminScope: appid
-      adminScopeParam: api://56c1bd14-3ba7-4804-b47b-d46de6dce33e/.default
+      issuerUrl: https://auth.sncloud-stg.dev/
+      issuerUrlParam: https://auth.sncloud-stg.dev/
+      audience: "urn:sn:pulsar:sndev:test"
+      audienceParam: "urn:sn:pulsar:sndev:test"
+      brokerClientCredential: '{"client_id":"CLIENT_ID","client_secret":"CLIENT_SECRET","issuer_url":"https://auth.sncloud-stg.dev/"}'
+      subjectClaim: https://streamnative.io/username
+      adminScope: https://streamnative.io/username
+      adminScopeParam: ""
       authenticationProviders: io.streamnative.pulsar.broker.authentication.AuthenticationProviderOAuth
diff --git a/.ci/examples/connect/pubsub.nar b/.ci/examples/connect/pubsub.nar
diff --git a/.ci/helm.sh b/.ci/helm.sh
@@ -598,7 +598,7 @@ function ci::verify_log_topic_with_auth() {
 }
 
 function ci::verify_env() {
-  pod="$1-function-0"
+  pod=$1
   key=$2
   expect=$3
   result=$(kubectl exec -n ${NAMESPACE} ${pod} -- env | grep "${key}")
@@ -608,4 +608,27 @@ function ci::verify_env() {
     return 0
   fi
   return 1
+}
+
+function ci::verify_liveness_probe() {
+  pod=$1
+  expected=$2
+  result=$(kubectl get pod $pod -o jsonpath='{.spec.containers[*].livenessProbe}')
+  echo "liveness probe is $result"
+  if [[ "$result" != "$expected" ]]; then
+    echo "failed"
+    return 1
+  fi
+  echo "succeeded"
+}
+
+function ci::verify_pod_log() {
+  pod=$1
+  log=$2
+  sleep 30
+  kubectl logs --tail=-1 $pod | grep "$log"
+  while [[ $? -ne 0 ]]; do
+      sleep 5
+      kubectl logs --tail=-1 $pod | grep "$log"
+  done
 }
diff --git a/.ci/tests/integration-oauth2/cases/batch-source/manifests.yaml b/.ci/tests/integration-oauth2/cases/batch-source/manifests.yaml
@@ -37,16 +37,14 @@ spec:
       certSecretKey: ""
     authConfig:
       oauth2Config:
-        audience: api://56c1bd14-3ba7-4804-b47b-d46de6dce33e/.default
-        issuerUrl: https://sts.windows.net/06a8a086-ae6e-45b5-a22e-ad90de23013e/v2.0
-        scope: api://56c1bd14-3ba7-4804-b47b-d46de6dce33e/.default
+        audience: urn:sn:pulsar:sndev:test
+        issuerUrl: https://auth.sncloud-stg.dev/
         keySecretName: sn-platform-oauth2-private-key
         keySecretKey: auth.json
     cleanupAuthConfig:
       oauth2Config:
-        audience: api://56c1bd14-3ba7-4804-b47b-d46de6dce33e/.default
-        issuerUrl: https://sts.windows.net/06a8a086-ae6e-45b5-a22e-ad90de23013e/v2.0
-        scope: api://56c1bd14-3ba7-4804-b47b-d46de6dce33e/.default
+        audience: urn:sn:pulsar:sndev:test
+        issuerUrl: https://auth.sncloud-stg.dev/
         keySecretName: sn-platform-oauth2-private-key
         keySecretKey: auth.json
   image: streamnative/pulsar-io-batch-data-generator:3.2.2.1

diff --git a/.../integration-oauth2/cases/global-and-namespaced-config-without-auto-update/manifests.yaml b/.../integration-oauth2/cases/global-and-namespaced-config-without-auto-update/manifests.yaml
@@ -0,0 +1,47 @@
+apiVersion: compute.functionmesh.io/v1alpha1
+kind: Source
+metadata:
+  name: test-datagen-source
+spec:
+  className: org.apache.pulsar.io.datagenerator.DataGeneratorSource
+  clusterName: test-pulsar
+  forwardSourceMessageProperty: true
+  image: docker.io/streamnative/pulsar-io-data-generator:3.2.2.1
+  java:
+    extraDependenciesDir: /pulsar/lib
+    jar: connectors/pulsar-io-data-generator-3.2.2.1.nar
+  minReplicas: 1
+  name: test-datagen-source
+  namespace: default
+  output:
+    producerConf: {}
+    topic: public/default/test-datagen-source
+    typeClassName: org.apache.pulsar.io.datagenerator.Person
+  processingGuarantee: atleast_once
+  pulsar:
+    authConfig:
+      oauth2Config:
+        audience: urn:sn:pulsar:sndev:test
+        issuerUrl: https://auth.sncloud-stg.dev/
+        keySecretName: sn-platform-oauth2-private-key
+        keySecretKey: auth.json
+    pulsarConfig: test-source
+  replicas: 1
+  resources:
+    limits:
+      cpu: "0.2"
+      memory: 1.1G
+    requests:
+      cpu: "0.1"
+      memory: 1G
+  sourceConfig:
+    sleepBetweenMessages: "5000"
+  tenant: public
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-source
+data:
+  webServiceURL: http://sn-platform-pulsar-broker.default.svc.cluster.local:8080
+  brokerServiceURL: pulsar://sn-platform-pulsar-broker.default.svc.cluster.local:6650
diff --git a/...ntegration-oauth2/cases/global-and-namespaced-config-without-auto-update/mesh-config.yaml b/...ntegration-oauth2/cases/global-and-namespaced-config-without-auto-update/mesh-config.yaml
@@ -0,0 +1,14 @@
+apiVersion: compute.functionmesh.io/v1alpha1
+kind: BackendConfig
+metadata:
+  name: backend-config
+  namespace: default
+spec:
+  env:
+    namespaced1: namespacedvalue1
+    shared1: fromnamespace
+    podenv: backendconfigvalue
+  pod:
+    liveness:
+      initialDelaySeconds: 30
+      periodSeconds: 10
diff --git a/...tests/integration-oauth2/cases/global-and-namespaced-config-without-auto-update/verify.sh b/...tests/integration-oauth2/cases/global-and-namespaced-config-without-auto-update/verify.sh
@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+E2E_DIR=$(dirname "$0")
+BASE_DIR=$(cd "${E2E_DIR}"/../../../../..;pwd)
+PULSAR_NAMESPACE=${PULSAR_NAMESPACE:-"default"}
+PULSAR_RELEASE_NAME=${PULSAR_RELEASE_NAME:-"sn-platform"}
+E2E_KUBECONFIG=${E2E_KUBECONFIG:-"/tmp/e2e-k8s.config"}
+
+source "${BASE_DIR}"/.ci/helm.sh
+
+if [ ! "$KUBECONFIG" ]; then
+  export KUBECONFIG=${E2E_KUBECONFIG}
+fi
+
+manifests_file="${BASE_DIR}"/.ci/tests/integration-oauth2/cases/global-and-namespaced-config-without-auto-update/manifests.yaml
+mesh_config_file="${BASE_DIR}"/.ci/tests/integration-oauth2/cases/global-and-namespaced-config-without-auto-update/mesh-config.yaml
+global_mesh_config_file="${BASE_DIR}"/.ci/clusters/global_backend_config.yaml
+
+kubectl apply -f "${mesh_config_file}" > /dev/null 2>&1
+kubectl apply -f "${manifests_file}" > /dev/null 2>&1
+
+verify_fm_result=$(ci::verify_function_mesh test-datagen-source 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_fm_result"
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" global1 global1=globalvalue1 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" namespaced1 namespaced1=namespacedvalue1 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+# if global and namespaced config has same key, the value from namespace should be used
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" shared1 shared1=fromnamespace 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+# verify liveness config
+verify_liveness_result=$(ci::verify_liveness_probe test-datagen-source-source-0 '{"failureThreshold":3,"httpGet":{"path":"/","port":9094,"scheme":"HTTP"},"initialDelaySeconds":30,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":10}' 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_liveness_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+# update the namespaced config, it should not trigger the reconcile since the autoUpdate is false
+kubectl patch BackendConfig backend-config --type='json' -p='[{"op": "replace", "path": "/spec/env/shared1", "value": "newvalue"}]' > /dev/null 2>&1
+sleep 30
+
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" shared1 shared1=fromnamespace 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+# delete the namespaced config, the source should not be reconciled since the autoUpdate is false
+kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+sleep 30
+
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" namespaced1 namespaced1=namespacedvalue1 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" shared1 shared1=fromnamespace 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+verify_liveness_result=$(ci::verify_liveness_probe test-datagen-source-source-0 '{"failureThreshold":3,"httpGet":{"path":"/","port":9094,"scheme":"HTTP"},"initialDelaySeconds":30,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":10}' 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_liveness_result"
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+# delete the global config, the source should be reconciled since the autoUpdate is true in the global config
+kubectl delete -f "${global_mesh_config_file}" -n $FUNCTION_MESH_NAMESPACE > /dev/null 2>&1 || true
+sleep 30
+
+verify_fm_result=$(ci::verify_function_mesh test-datagen-source 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_fm_result"
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" global1 "" 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" namespaced1 "" 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+verify_env_result=$(ci::verify_env "test-datagen-source-source-0" shared1 "" 2>&1)
+if [ $? -ne 0 ]; then
+  echo "$verify_env_result"
+  kubectl delete -f "${mesh_config_file}" > /dev/null 2>&1
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+# it should use liveness config from namespaced config
+verify_liveness_result=$(ci::verify_liveness_probe test-datagen-source-source-0 "" 2>&1)
+if [ $? -eq 0 ]; then
+  echo "e2e-test: ok" | yq eval -
+else
+  echo "$verify_liveness_result"
+  kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
+  exit 1
+fi
+
+kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true
diff --git a/.ci/tests/integration-oauth2/cases/global-and-namespaced-config/manifests.yaml b/.ci/tests/integration-oauth2/cases/global-and-namespaced-config/manifests.yaml
@@ -0,0 +1,50 @@
+apiVersion: compute.functionmesh.io/v1alpha1
+kind: Sink
+metadata:
+  name: test-datagen-sink
+spec:
+  autoAck: true
+  className: org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink
+  clusterName: test-pulsar
+  image: docker.io/streamnative/pulsar-io-data-generator:3.2.2.1
+  input:
+    sourceSpecs:
+      public/default/datagen:
+        receiverQueueSize: 1000
+    topics:
+      - public/default/datagen
+    typeClassName: org.apache.pulsar.io.datagenerator.Person
+  java:
+    extraDependenciesDir: /pulsar/lib
+    jar: connectors/pulsar-io-data-generator-3.2.2.1.nar
+  minReplicas: 1
+  namespace: default
+  processingGuarantee: atleast_once
+  sinkConfig: {}
+  pulsar:
+    authConfig:
+      oauth2Config:
+        audience: urn:sn:pulsar:sndev:test
+        issuerUrl: https://auth.sncloud-stg.dev/
+        keySecretName: sn-platform-oauth2-private-key
+        keySecretKey: auth.json
+    pulsarConfig: test-sink
+  replicas: 1
+  resources:
+    limits:
+      cpu: "0.2"
+      memory: 1.1G
+    requests:
+      cpu: "0.1"
+      memory: 1G
+  subscriptionName: mysub
+  subscriptionPosition: latest
+  tenant: public
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-sink
+data:
+  webServiceURL: http://sn-platform-pulsar-broker.default.svc.cluster.local:8080
+  brokerServiceURL: pulsar://sn-platform-pulsar-broker.default.svc.cluster.local:6650
diff --git a/.ci/tests/integration-oauth2/cases/global-and-namespaced-config/mesh-config-kube-system.yaml b/.ci/tests/integration-oauth2/cases/global-and-namespaced-config/mesh-config-kube-system.yaml
@@ -0,0 +1,11 @@
+apiVersion: compute.functionmesh.io/v1alpha1
+kind: BackendConfig
+metadata:
+  name: backend-config
+  namespace: kube-system
+spec:
+  autoUpdate: true
+  pod:
+    liveness:
+      initialDelaySeconds: 50
+      periodSeconds: 60