Merge pull request #9 from deepthi912/master

reverse merge
deepthi912 · May 1, 2024 · 4f5d25f · 4f5d25f
2 parents f3ae8fb + 5406079
commit 4f5d25f
Show file tree

Hide file tree

Showing 142 changed files with 2,589 additions and 2,079 deletions.
diff --git a/LICENSE-binary b/LICENSE-binary
@@ -371,8 +371,8 @@ jakarta.validation:jakarta.validation-api:2.0.2
 javax.inject:javax.inject:1
 javax.validation:validation-api:2.0.1.Final
 joda-time:joda-time:2.12.5
-net.java.dev.jna:jna-platform:5.6.0
-net.java.dev.jna:jna:5.5.0
+net.java.dev.jna:jna-platform:5.14.0
+net.java.dev.jna:jna:5.14.0
 net.minidev:accessors-smart:2.5.0
 net.minidev:json-smart:2.5.0
 net.openhft:chronicle-analytics:2.24ea0
@@ -420,7 +420,7 @@ org.apache.helix:helix-core:1.3.1
 org.apache.helix:metadata-store-directory-common:1.3.1
 org.apache.helix:metrics-common:1.3.1
 org.apache.helix:zookeeper-api:1.3.1
-org.apache.hive:hive-storage-api:2.7.1
+org.apache.hive:hive-storage-api:2.8.1
 org.apache.httpcomponents:httpclient:4.5.14
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.httpcomponents:httpmime:4.5.13
@@ -438,8 +438,8 @@ org.apache.lucene:lucene-core:9.8.0
 org.apache.lucene:lucene-queries:9.8.0
 org.apache.lucene:lucene-queryparser:9.8.0
 org.apache.lucene:lucene-sandbox:9.8.0
-org.apache.orc:orc-core:1.5.9
-org.apache.orc:orc-shims:1.5.9
+org.apache.orc:orc-core:1.9.3
+org.apache.orc:orc-shims:1.9.3
 org.apache.parquet:parquet-avro:1.13.1
 org.apache.parquet:parquet-column:1.13.1
 org.apache.parquet:parquet-common:1.13.1

diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
@@ -85,6 +85,9 @@ rules:
 - pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ControllerMetrics\", name=\"pinot\\.controller\\.offlineTableCount\"><>(\\w+)"
   name: "pinot_controller_offlineTableCount_$1"
   cache: true
+- pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ControllerMetrics\", name=\"pinot\\.controller\\.tierBackendTableCount\\.(\\w+)\"><>(\\w+)"
+  name: "pinot_controller_tierBackendTableCount_$1_$2"
+  cache: true
 - pattern: "\"org\\.apache\\.pinot\\.common\\.metrics\"<type=\"ValidationMetrics\", name=\"pinot\\.controller\\.(([^.]+)\\.)?([^.]*)\\.(\\w+)\"><>(\\w+)"
   name: "pinot_controller_validateion_$4_$5"
   cache: true

diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/querylog/QueryLogger.java b/pinot-broker/src/main/java/org/apache/pinot/broker/querylog/QueryLogger.java
@@ -98,7 +98,7 @@ public void log(QueryLogParams params) {
       long numDroppedLogsSinceLastLog = _numDroppedLogs.getAndSet(0);
       if (numDroppedLogsSinceLastLog > 0) {
         _logger.warn("{} logs were dropped. (log max rate per second: {})", numDroppedLogsSinceLastLog,
-            _droppedLogRateLimiter.getRate());
+            _logRateLimiter.getRate());
       }
     }
   }

diff --git a/...broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java b/...broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
@@ -24,7 +24,6 @@
 import com.google.common.collect.ImmutableMap;
 import java.net.URI;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -1361,17 +1360,15 @@ private static void handleApproximateFunctionOverride(Expression expression) {
         try {
           int percentile = Integer.parseInt(remainingFunctionName);
           function.setOperator("percentilesmarttdigest");
-          function.setOperands(
-              Arrays.asList(function.getOperands().get(0), RequestUtils.getLiteralExpression(percentile)));
+          function.addToOperands(RequestUtils.getLiteralExpression(percentile));
         } catch (Exception e) {
           throw new BadQueryRequestException("Illegal function name: " + functionName);
         }
       } else if (remainingFunctionName.matches("\\d+mv")) {
         try {
           int percentile = Integer.parseInt(remainingFunctionName.substring(0, remainingFunctionName.length() - 2));
           function.setOperator("percentilesmarttdigest");
-          function.setOperands(
-              Arrays.asList(function.getOperands().get(0), RequestUtils.getLiteralExpression(percentile)));
+          function.addToOperands(RequestUtils.getLiteralExpression(percentile));
         } catch (Exception e) {
           throw new BadQueryRequestException("Illegal function name: " + functionName);
         }
@@ -1849,18 +1846,17 @@ static void validateRequest(PinotQuery pinotQuery, int queryResponseLimit) {
    */
   private static void attachTimeBoundary(PinotQuery pinotQuery, TimeBoundaryInfo timeBoundaryInfo,
       boolean isOfflineRequest) {
+    String functionName = isOfflineRequest ? FilterKind.LESS_THAN_OR_EQUAL.name() : FilterKind.GREATER_THAN.name();
     String timeColumn = timeBoundaryInfo.getTimeColumn();
     String timeValue = timeBoundaryInfo.getTimeValue();
-    Expression timeFilterExpression = RequestUtils.getFunctionExpression(
-        isOfflineRequest ? FilterKind.LESS_THAN_OR_EQUAL.name() : FilterKind.GREATER_THAN.name());
-    timeFilterExpression.getFunctionCall().setOperands(
-        Arrays.asList(RequestUtils.getIdentifierExpression(timeColumn), RequestUtils.getLiteralExpression(timeValue)));
+    Expression timeFilterExpression =
+        RequestUtils.getFunctionExpression(functionName, RequestUtils.getIdentifierExpression(timeColumn),
+            RequestUtils.getLiteralExpression(timeValue));
 
     Expression filterExpression = pinotQuery.getFilterExpression();
     if (filterExpression != null) {
-      Expression andFilterExpression = RequestUtils.getFunctionExpression(FilterKind.AND.name());
-      andFilterExpression.getFunctionCall().setOperands(Arrays.asList(filterExpression, timeFilterExpression));
-      pinotQuery.setFilterExpression(andFilterExpression);
+      pinotQuery.setFilterExpression(
+          RequestUtils.getFunctionExpression(FilterKind.AND.name(), filterExpression, timeFilterExpression));
     } else {
       pinotQuery.setFilterExpression(timeFilterExpression);
     }

diff --git a/.../src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java b/.../src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
@@ -170,6 +170,12 @@ protected BrokerResponse handleRequest(long requestId, String query, @Nullable S
 
     DispatchableSubPlan dispatchableSubPlan = queryPlanResult.getQueryPlan();
     Set<String> tableNames = queryPlanResult.getTableNames();
+
+    _brokerMetrics.addMeteredGlobalValue(BrokerMeter.MULTI_STAGE_QUERIES_GLOBAL, 1);
+    for (String tableName : tableNames) {
+      _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.MULTI_STAGE_QUERIES, 1);
+    }
+
     requestContext.setTableNames(List.copyOf(tableNames));
 
     // Compilation Time. This includes the time taken for parsing, compiling, create stage plans and assigning workers.

diff --git a/pinot-clients/pinot-jdbc-client/pom.xml b/pinot-clients/pinot-jdbc-client/pom.xml
@@ -33,7 +33,6 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/../..</pinot.root>
-    <shade.phase.prop>package</shade.phase.prop>
   </properties>
   <build>
     <resources>
@@ -82,4 +81,18 @@
       <artifactId>jsr305</artifactId>
     </dependency>
   </dependencies>
+  <profiles>
+    <profile>
+      <id>build-shaded-jar</id>
+      <activation>
+        <property>
+          <name>skipShade</name>
+          <value>!true</value>
+        </property>
+      </activation>
+      <properties>
+        <shade.phase.prop>package</shade.phase.prop>
+      </properties>
+    </profile>
+  </profiles>
 </project>
diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml
@@ -33,7 +33,6 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/..</pinot.root>
-    <shade.phase.prop>package</shade.phase.prop>
   </properties>
 
   <build>
@@ -63,6 +62,19 @@
         <artifactId>protobuf-maven-plugin</artifactId>
       </plugin>
 
+      <plugin>
+        <groupId>com.diffplug.spotless</groupId>
+        <artifactId>spotless-maven-plugin</artifactId>
+        <configuration>
+          <java>
+            <excludes>
+              <exclude>src/main/java/org/apache/pinot/common/request/*.java</exclude>
+              <exclude>src/main/java/org/apache/pinot/common/response/ProcessingException.java</exclude>
+            </excludes>
+          </java>
+        </configuration>
+      </plugin>
+
       <!-- Following plugins and their configurations are used to generate the custom Calcite's SQL parser -->
       <!-- Copy the templates present in the codegen directory to ${project.build.directory}/codegen -->
       <plugin>
@@ -130,31 +142,6 @@
         </executions>
       </plugin>
     </plugins>
-
-    <pluginManagement>
-      <plugins>
-        <plugin>
-          <groupId>com.diffplug.spotless</groupId>
-          <artifactId>spotless-maven-plugin</artifactId>
-          <configuration>
-            <java>
-              <includes>
-                <include>src/main/java/**/*.java</include>
-                <include>src/test/java/**/*.java</include>
-              </includes>
-              <excludes>
-                <exclude>src/main/java/org/apache/pinot/common/request/*.java</exclude>
-                <exclude>src/main/java/org/apache/pinot/common/response/ProcessingException.java</exclude>
-              </excludes>
-              <importOrder>
-                <order>,\#</order>
-              </importOrder>
-              <removeUnusedImports/>
-            </java>
-          </configuration>
-        </plugin>
-      </plugins>
-    </pluginManagement>
   </build>
   <dependencies>
     <dependency>
@@ -414,5 +401,15 @@
         </plugins>
       </build>
     </profile>
+
+    <profile>
+      <id>build-shaded-jar</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <properties>
+        <shade.phase.prop>package</shade.phase.prop>
+      </properties>
+    </profile>
   </profiles>
 </project>