Merge pull request #7 from apache/master

Reverse merge
deepthi912 · Apr 18, 2024 · f3ae8fb · f3ae8fb
2 parents d12b2d1 + 7a4c0b8
commit f3ae8fb
Show file tree

Hide file tree

Showing 106 changed files with 1,599 additions and 1,377 deletions.
diff --git a/.github/workflows/scripts/docker/.superset_docker_image_build_and_push.sh b/.github/workflows/scripts/docker/.superset_docker_image_build_and_push.sh
@@ -49,7 +49,7 @@ done
 
 cd ${DOCKER_FILE_BASE_DIR}
 
-docker buildx build \
+docker build \
     --no-cache \
     --platform=${BUILD_PLATFORM} \
     --file Dockerfile \

diff --git a/.github/workflows/scripts/docker/.superset_multi_arch_docker_image_build_and_push.sh b/.github/workflows/scripts/docker/.superset_multi_arch_docker_image_build_and_push.sh
@@ -0,0 +1,59 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+  DOCKER_IMAGE_NAME="apachepinot/pinot-superset"
+fi
+if [ -z "${SUPERSET_IMAGE_TAG}" ]; then
+  SUPERSET_IMAGE_TAG="latest"
+fi
+if [ -z "${BUILD_PLATFORM}" ]; then
+  BUILD_PLATFORM="linux/amd64"
+fi
+
+DATE=`date +%Y%m%d`
+docker pull apache/superset:${SUPERSET_IMAGE_TAG}
+COMMIT_ID=`docker images apache/superset:${SUPERSET_IMAGE_TAG} --format "{{.ID}}"`
+
+tags=()
+if [ -z "${TAGS}" ]; then
+  tags=("${COMMIT_ID}-${DATE}")
+  tags+=("latest")
+else
+  declare -a tags=($(echo ${TAGS} | tr "," " "))
+fi
+
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"
+do
+  echo "Plan to build and push docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
+  DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag} "
+done
+
+cd ${DOCKER_FILE_BASE_DIR}
+
+docker build \
+    --no-cache \
+    --platform=${BUILD_PLATFORM} \
+    --file Dockerfile \
+    --build-arg SUPERSET_IMAGE_TAG=${SUPERSET_IMAGE_TAG} \
+    ${DOCKER_BUILD_TAGS} \
+    --push \
+    .
diff --git a/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh b/.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
@@ -33,6 +33,7 @@ if [ "$RUN_TEST_SET" == "1" ]; then
       -pl 'pinot-spi' \
       -pl 'pinot-segment-spi' \
       -pl 'pinot-common' \
+      -pl ':pinot-yammer' \
       -pl 'pinot-core' \
       -pl 'pinot-query-planner' \
       -pl 'pinot-query-runtime' \
@@ -46,5 +47,6 @@ if [ "$RUN_TEST_SET" == "2" ]; then
     -pl '!pinot-core' \
     -pl '!pinot-query-planner' \
     -pl '!pinot-query-runtime' \
+    -pl '!:pinot-yammer' \
     -P github-actions,no-integration-tests || exit 1
 fi
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml
@@ -1,173 +1,173 @@
 rules:
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?).authorization\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.authorization\"><>(\\w+)"
   name: "pinot_broker_authorization_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.documentsScanned\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.documentsScanned\"><>(\\w+)"
   name: "pinot_broker_documentsScanned_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.entriesScannedInFilter\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.entriesScannedInFilter\"><>(\\w+)"
   name: "pinot_broker_entriesScannedInFilter_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.entriesScannedPostFilter\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.entriesScannedPostFilter\"><>(\\w+)"
   name: "pinot_broker_entriesScannedPostFilter_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.freshnessLagMs\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.freshnessLagMs\"><>(\\w+)"
   name: "pinot_broker_freshnessLagMs_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.queries\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.queries\"><>(\\w+)"
   name: "pinot_broker_queries_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.queryExecution\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.queryExecution\"><>(\\w+)"
   name: "pinot_broker_queryExecution_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.queryRouting\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.queryRouting\"><>(\\w+)"
   name: "pinot_broker_queryRouting_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.reduce\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.reduce\"><>(\\w+)"
   name: "pinot_broker_reduce_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.requestCompilation\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.requestCompilation\"><>(\\w+)"
   name: "pinot_broker_requestCompilation_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.scatterGather\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.scatterGather\"><>(\\w+)"
   name: "pinot_broker_scatterGather_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.totalServerResponseSize\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.totalServerResponseSize\"><>(\\w+)"
   name: "pinot_broker_totalServerResponseSize_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)_(OFFLINE|REALTIME).groupBySize\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)_(OFFLINE|REALTIME)\\.groupBySize\"><>(\\w+)"
   name: "pinot_broker_groupBySize_$5"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
     tableType: "$4"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)_(OFFLINE|REALTIME).noServingHostForSegment\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)_(OFFLINE|REALTIME)\\.noServingHostForSegment\"><>(\\w+)"
   name: "pinot_broker_noServingHostForSegment_$5"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
     tableType: "$4"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.healthcheck(\\w+)\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.healthcheck(\\w+)\"><>(\\w+)"
   name: "pinot_broker_healthcheck_$1_$2"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.helix.(\\w+)\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.helix\\.(\\w+)\"><>(\\w+)"
   name: "pinot_broker_helix_$1_$2"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.helixZookeeper(\\w+)\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.helixZookeeper(\\w+)\"><>(\\w+)"
   name: "pinot_broker_helix_zookeeper_$1_$2"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.nettyConnection(\\w+)\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.nettyConnection(\\w+)\"><>(\\w+)"
   name: "pinot_broker_nettyConnection_$1_$2"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.unhealthyServers\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.unhealthyServers\"><>(\\w+)"
   name: "pinot_broker_unhealthyServers_$1"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.clusterChangeCheck\"\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.clusterChangeCheck\"\"><>(\\w+)"
   name: "pinot_broker_clusterChangeCheck_$1"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.proactiveClusterChangeCheck\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.proactiveClusterChangeCheck\"><>(\\w+)"
   name: "pinot_broker_proactiveClusterChangeCheck_$1"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(\\w+)Exceptions\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(\\w+)Exceptions\"><>(\\w+)"
   name: "pinot_broker_exceptions_$1_$2"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.routingTableUpdateTime\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.routingTableUpdateTime\"><>(\\w+)"
   name: "pinot_broker_routingTableUpdateTime_$1"
   cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.brokerResponsesWithPartialServersResponded\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.brokerResponsesWithPartialServersResponded\"><>(\\w+)"
   name: "pinot_broker_brokerResponsesWithPartialServersResponded_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.brokerResponsesWithTimeouts\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.brokerResponsesWithTimeouts\"><>(\\w+)"
   name: "pinot_broker_brokerResponsesWithTimeouts_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.noServerFoundExceptions\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.noServerFoundExceptions\"><>(\\w+)"
   name: "pinot_broker_noServerFoundExceptions_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.brokerResponsesWithProcessingExceptions\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.brokerResponsesWithProcessingExceptions\"><>(\\w+)"
   name: "pinot_broker_brokerResponsesWithProcessingExceptions_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.brokerResponsesWithNumGroupsLimitReached\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.brokerResponsesWithNumGroupsLimitReached\"><>(\\w+)"
   name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.queryQuotaExceeded\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.queryQuotaExceeded\"><>(\\w+)"
   name: "pinot_broker_queryQuotaExceeded_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.queryTotalTimeMs\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.queryTotalTimeMs\"><>(\\w+)"
   name: "pinot_broker_queryTotalTimeMs_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)_(OFFLINE|REALTIME).serverMissingForRouting\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)_(OFFLINE|REALTIME)\\.serverMissingForRouting\"><>(\\w+)"
   name: "pinot_broker_serverMissingForRouting_$5"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
     tableType: "$4"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.deserialization\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.deserialization\"><>(\\w+)"
   name: "pinot_broker_deserialization_$4"
   cache: true
   labels:
     database: "$2"
     table: "$1$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<type=\"BrokerMetrics\", name=\"pinot.broker.(([^\\.]+)\\.)?([^\\.]*?)\\.requestConnectionWait\"><>(\\w+)"
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"BrokerMetrics\", name=\"pinot\\.broker\\.(([^.]+)\\.)?([^.]*)\\.requestConnectionWait\"><>(\\w+)"
   name: "pinot_broker_requestConnectionWait_$4"
   cache: true
   labels:
@@ -207,7 +207,7 @@ rules:
     table: "$3$5"
     tableType: "$6"
   #This is a catch-all pattern for pinot table metrics with offline/realtime suffix that also contain kafka topic
-- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<type=\"?\\w+\"?, name=\"?pinot\\.(\\w+)\\.(\\w+)\\.((\\w+)\\.)?(\\w+)_(OFFLINE|REALTIME)\\-(.+)\\-(\\w+)\"?><>(\\w+)"
+- pattern: "\"?org\\.apache\\.pinot\\.common\\.metrics\"?<type=\"?\\w+\"?, name=\"?pinot\\.(\\w+)\\.(\\w+)\\.((\\w+)\\.)?(\\w+)_(OFFLINE|REALTIME)\\-(\\.+)\\-(\\w+)\"?><>(\\w+)"
   name: "pinot_$1_$2_$9"
   cache: true
   labels: