From f2e04fc27d161fb138ec51c22826469fcbea80ad Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Tue, 26 Nov 2024 14:57:45 +0800 Subject: [PATCH] Change field processingGuarantee to processingGuarantees for WindowConfig (#794) * Change field processingGuarantee to processingGuarantees for WindowConfig * fix ci --- .ci/helm.sh | 2 ++ .ci/tests/integration-oauth2/e2e.yaml | 1 + .ci/tests/integration-oauth2/e2e_with_downloader.yaml | 1 + .../cases/logging-window-function/manifests.yaml | 2 +- .../cases/logging-window-function/verify.sh | 10 +++++++++- .ci/tests/integration/e2e.yaml | 1 + .ci/tests/integration/e2e_with_tls.yaml | 1 + api/compute/v1alpha1/common.go | 2 +- .../crd-compute.functionmesh.io-functionmeshes.yaml | 2 +- .../crd-compute.functionmesh.io-functions.yaml | 2 +- .../bases/compute.functionmesh.io_functionmeshes.yaml | 2 +- .../crd/bases/compute.functionmesh.io_functions.yaml | 2 +- 12 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.ci/helm.sh b/.ci/helm.sh index 0a16f40f..51782a11 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -92,6 +92,8 @@ function ci::install_pulsar_charts() { helm repo add grafana https://grafana.github.io/helm-charts helm repo update yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml + # the superset chart seems got some issue, so we remove it + yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml helm dependency update charts/pulsar ${HELM} install sn-platform --set initialize=true --values charts/pulsar/mini_values.yaml charts/pulsar --debug diff --git a/.ci/tests/integration-oauth2/e2e.yaml b/.ci/tests/integration-oauth2/e2e.yaml index 4003041e..6d4f7b66 100644 --- a/.ci/tests/integration-oauth2/e2e.yaml +++ b/.ci/tests/integration-oauth2/e2e.yaml @@ -39,6 +39,7 @@ setup: helm repo add grafana https://grafana.github.io/helm-charts helm repo update yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml + yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml helm dependency update charts/pulsar helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster_with_oauth.yaml charts/pulsar diff --git a/.ci/tests/integration-oauth2/e2e_with_downloader.yaml b/.ci/tests/integration-oauth2/e2e_with_downloader.yaml index b6833c07..5474c745 100644 --- a/.ci/tests/integration-oauth2/e2e_with_downloader.yaml +++ b/.ci/tests/integration-oauth2/e2e_with_downloader.yaml @@ -39,6 +39,7 @@ setup: helm repo add grafana https://grafana.github.io/helm-charts helm repo update yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml + yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml helm dependency update charts/pulsar helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster_with_oauth.yaml charts/pulsar diff --git a/.ci/tests/integration/cases/logging-window-function/manifests.yaml b/.ci/tests/integration/cases/logging-window-function/manifests.yaml index 0d6b77f8..d030c826 100644 --- a/.ci/tests/integration/cases/logging-window-function/manifests.yaml +++ b/.ci/tests/integration/cases/logging-window-function/manifests.yaml @@ -36,7 +36,7 @@ spec: windowConfig: windowLengthCount: 10 slidingIntervalCount: 5 - processingGuarantee: ATLEAST_ONCE + processingGuarantees: ATLEAST_ONCE # the processingGuarantee should be manual for window function # see: https://github.com/apache/pulsar/pull/16279/files#diff-c77c024ccb31c94a7aa80cb8e96d7e370709157bdc104a1be7867fb6c7aa0586R318-R319 processingGuarantee: manual diff --git a/.ci/tests/integration/cases/logging-window-function/verify.sh b/.ci/tests/integration/cases/logging-window-function/verify.sh index 8828be91..eb5f4ae3 100644 --- a/.ci/tests/integration/cases/logging-window-function/verify.sh +++ b/.ci/tests/integration/cases/logging-window-function/verify.sh @@ -43,6 +43,14 @@ if [ $? -ne 0 ]; then exit 1 fi +# verify the `processingGuarantees` config +verify_pg=$(kubectl logs window-function-sample-function-0 | grep processingGuarantees=ATLEAST_ONCE) +if [ $? -ne 0 ]; then + echo "$verify_pg" + kubectl delete -f "${manifests_file}" > /dev/null 2>&1 || true + exit 1 +fi + verify_java_result=$(NAMESPACE=${PULSAR_NAMESPACE} CLUSTER=${PULSAR_RELEASE_NAME} ci::send_test_data "persistent://public/default/window-function-input-topic" "test-message" 3 2>&1) if [ $? -ne 0 ]; then echo "$verify_java_result" @@ -80,7 +88,7 @@ fi #fi verify_log_result=$(kubectl logs -l compute.functionmesh.io/name=window-function-sample --tail=-1 | grep -e "-window-log" | wc -l) -if [ $verify_log_result -ne 0 ]; then +if [ $verify_log_result -eq 15 ]; then sub_name=$(echo $RANDOM | md5sum | head -c 20; echo;) verify_log_topic_result=$(kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-broker-0 -- bin/pulsar-client consume -n 15 -s $sub_name --subscription-position Earliest "persistent://public/default/window-function-logs" | grep -e "-window-log" | wc -l) if [ $verify_log_topic_result -ne 0 ]; then diff --git a/.ci/tests/integration/e2e.yaml b/.ci/tests/integration/e2e.yaml index ed402063..73cedf4d 100644 --- a/.ci/tests/integration/e2e.yaml +++ b/.ci/tests/integration/e2e.yaml @@ -45,6 +45,7 @@ setup: helm repo add grafana https://grafana.github.io/helm-charts helm repo update yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml + yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml helm dependency update charts/pulsar helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster.yaml charts/pulsar diff --git a/.ci/tests/integration/e2e_with_tls.yaml b/.ci/tests/integration/e2e_with_tls.yaml index 2182ba2d..5aecd064 100644 --- a/.ci/tests/integration/e2e_with_tls.yaml +++ b/.ci/tests/integration/e2e_with_tls.yaml @@ -34,6 +34,7 @@ setup: helm repo add grafana https://grafana.github.io/helm-charts helm repo update yq -i '.dependencies[0].repository = "https://grafana.github.io/helm-charts"' charts/pulsar/requirements.yaml + yq -i 'del(.dependencies[1])' charts/pulsar/requirements.yaml helm dependency update charts/pulsar helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster_with_tls.yaml charts/pulsar diff --git a/api/compute/v1alpha1/common.go b/api/compute/v1alpha1/common.go index 178fdfeb..f679038a 100644 --- a/api/compute/v1alpha1/common.go +++ b/api/compute/v1alpha1/common.go @@ -546,7 +546,7 @@ type WindowConfig struct { MaxLagMs *int64 `json:"maxLagMs,omitempty"` WatermarkEmitIntervalMs *int64 `json:"watermarkEmitIntervalMs,omitempty"` TimestampExtractorClassName *string `json:"timestampExtractorClassName,omitempty"` - ProcessingGuarantee WindowProcessGuarantee `json:"processingGuarantee,omitempty"` + ProcessingGuarantees WindowProcessGuarantee `json:"processingGuarantees,omitempty"` } type VPASpec struct { diff --git a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functionmeshes.yaml b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functionmeshes.yaml index aad5b9d8..e48ac2a2 100644 --- a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functionmeshes.yaml +++ b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functionmeshes.yaml @@ -3786,7 +3786,7 @@ spec: maxLagMs: format: int64 type: integer - processingGuarantee: + processingGuarantees: enum: - ATLEAST_ONCE - ATMOST_ONCE diff --git a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functions.yaml b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functions.yaml index 179f9ebd..5289962a 100644 --- a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functions.yaml +++ b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functions.yaml @@ -3805,7 +3805,7 @@ spec: maxLagMs: format: int64 type: integer - processingGuarantee: + processingGuarantees: enum: - ATLEAST_ONCE - ATMOST_ONCE diff --git a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml index 890541ce..5deb3eb1 100644 --- a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml +++ b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml @@ -3786,7 +3786,7 @@ spec: maxLagMs: format: int64 type: integer - processingGuarantee: + processingGuarantees: enum: - ATLEAST_ONCE - ATMOST_ONCE diff --git a/config/crd/bases/compute.functionmesh.io_functions.yaml b/config/crd/bases/compute.functionmesh.io_functions.yaml index f1546efc..a9e28d8e 100644 --- a/config/crd/bases/compute.functionmesh.io_functions.yaml +++ b/config/crd/bases/compute.functionmesh.io_functions.yaml @@ -3783,7 +3783,7 @@ spec: maxLagMs: format: int64 type: integer - processingGuarantee: + processingGuarantees: enum: - ATLEAST_ONCE - ATMOST_ONCE