reverse merge

.asf.yaml

@@ -31,12 +31,12 @@ github:
     rebase: false
   collaborators:
     - sullis
-    - shenyu0127
-    - tibrewalpratik17
+    - yashmayya
     - abhioncbr
-    - zhtaoxiang
     - shounakmk219
     - itschrispeck
+    - zhtaoxiang
     - soumitra-st
     - swaminathanmanish
-    - yashmayya
+    - suddendust
+    - vrajat

.github/dependabot.yml

@@ -25,6 +25,19 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 20
+    ignore:
+    - dependency-name: "org.apache.pinot:pinot-dropwizard"
+      # Locked test resource for pinot-spi/src/test/java/org/apache/pinot/spi/plugin/ClassLoaderTest.java
+      versions: ["0.10.0"]
+    - dependency-name: "org.apache.pinot:pinot-yammer"
+      # Locked test resource for pinot-spi/src/test/java/org/apache/pinot/spi/plugin/ClassLoaderTest.java
+      versions: ["0.10.0"]
+    - dependency-name: "commons-io:commons-io"
+      # Locked test resource for pinot-spi/src/test/java/org/apache/pinot/spi/plugin/ClassLoaderTest.java
+      versions: ["2.11.0"]
+    - dependency-name: "com.yammer.metrics:metrics-core"
+      # Locked test resource for pinot-spi/src/test/java/org/apache/pinot/spi/plugin/ClassLoaderTest.java
+      versions: ["2.1.5"]
 
   - package-ecosystem: "npm"
     directory: "/pinot-controller/src/main/resources"

.github/workflows/pinot_tests.yml

@@ -123,6 +123,11 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+            --add-opens=java.base/java.nio=ALL-UNNAMED
+            --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+            --add-opens=java.base/java.lang=ALL-UNNAMED
+            --add-opens=java.base/java.util=ALL-UNNAMED
+            --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
         run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
       - name: Unit Test
         env:
@@ -140,6 +145,11 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+            --add-opens=java.base/java.nio=ALL-UNNAMED
+            --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+            --add-opens=java.base/java.lang=ALL-UNNAMED
+            --add-opens=java.base/java.util=ALL-UNNAMED
+            --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
         run: .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
@@ -267,7 +277,7 @@ jobs:
       matrix:
         test_suite: [ "compatibility-verifier/sample-test-suite" ]
         old_commit: [
-          "release-1.0.0", "release-1.1.0", "master"
+          "release-1.0.0", "release-1.2.0", "master"
         ]
     name: Pinot Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
     steps:
@@ -323,7 +333,7 @@ jobs:
       matrix:
         test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ]
         old_commit: [
-          "master"
+          "release-1.2.0", "master"
         ]
     name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
     steps:

.github/workflows/scripts/.pinot_vuln_check.sh

@@ -40,4 +40,8 @@ docker build \
     --tag ${DOCKER_IMAGE_NAME}:${PINOT_SHA} \
     .
 
+docker builder prune -a -f
+
+docker image prune -f
+
 docker image ls

.github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh

@@ -29,5 +29,5 @@ netstat -i
 cd pinot-integration-tests || exit 1
 if [ "$RUN_TEST_SET" == "1" ]; then
   mvn test \
-      -P github-actions,custom-cluster-integration-test-suite || exit 1
+      -P github-actions,codecoverage,custom-cluster-integration-test-suite || exit 1
 fi

.github/workflows/scripts/pr-tests/.pinot_tests_integration.sh

@@ -29,9 +29,9 @@ netstat -i
 cd pinot-integration-tests || exit 1
 if [ "$RUN_TEST_SET" == "1" ]; then
   mvn test \
-      -P github-actions,integration-tests-set-1 && exit 0 || exit 1
+      -P github-actions,codecoverage,integration-tests-set-1 && exit 0 || exit 1
 fi
 if [ "$RUN_TEST_SET" == "2" ]; then
   mvn test \
-      -P github-actions,integration-tests-set-2 && exit 0 || exit 1
+      -P github-actions,codecoverage,integration-tests-set-2 && exit 0 || exit 1
 fi

.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh

@@ -37,7 +37,7 @@ if [ "$RUN_TEST_SET" == "1" ]; then
       -pl 'pinot-core' \
       -pl 'pinot-query-planner' \
       -pl 'pinot-query-runtime' \
-      -P github-actions,no-integration-tests || exit 1
+      -P github-actions,codecoverage,no-integration-tests || exit 1
 fi
 if [ "$RUN_TEST_SET" == "2" ]; then
   mvn test -T 16 \
@@ -48,5 +48,7 @@ if [ "$RUN_TEST_SET" == "2" ]; then
     -pl '!pinot-query-planner' \
     -pl '!pinot-query-runtime' \
     -pl '!:pinot-yammer' \
-    -P github-actions,no-integration-tests || exit 1
+    -P github-actions,codecoverage,no-integration-tests || exit 1
 fi
+
+mvn jacoco:report-aggregate@report -P codecoverage || exit 1

.gitignore

@@ -45,7 +45,8 @@ yarn-error.log*
 quickstart*
 
 #build symlink directory
-build*
+build
+build/*
 
 #helm related files
 kubernetes/helm/**/charts/

.mvn/extensions.xml

@@ -28,6 +28,6 @@
     <extension>
         <groupId>com.gradle</groupId>
         <artifactId>common-custom-user-data-maven-extension</artifactId>
-        <version>2</version>
+        <version>2.0.1</version>
     </extension>
 </extensions>

LICENSE-binary

@@ -279,7 +279,7 @@ com.typesafe.netty:netty-reactive-streams:2.0.4
 com.typesafe.scala-logging:scala-logging_2.12:3.9.2
 com.uber:h3:4.1.1
 com.yammer.metrics:metrics-core:2.2.0
-com.yscope.clp:clp-ffi:0.4.4
+com.yscope.clp:clp-ffi:0.4.5
 com.zaxxer:HikariCP-java7:2.4.13
 commons-cli:commons-cli:1.8.0
 commons-codec:commons-codec:1.17.0
@@ -361,10 +361,10 @@ io.opencensus:opencensus-proto:0.2.0
 io.opentelemetry:opentelemetry-api-incubator:1.37.0-alpha
 io.opentelemetry:opentelemetry-api:1.37.0
 io.opentelemetry:opentelemetry-context:1.37.0
-io.perfmark:perfmark-api:0.27.0
+io.perfmark:perfmark-api:0.26.0
 io.projectreactor.netty:reactor-netty-core:1.0.45
 io.projectreactor.netty:reactor-netty-http:1.0.45
-io.projectreactor:reactor-core:3.4.48
+io.projectreactor:reactor-core:3.4.38
 io.swagger.core.v3:swagger-annotations:2.1.10
 io.swagger:swagger-annotations:1.6.14
 io.swagger:swagger-core:1.6.14
@@ -487,8 +487,8 @@ org.apache.yetus:audience-annotations:0.15.0
 org.apache.zookeeper:zookeeper-jute:3.9.2
 org.apache.zookeeper:zookeeper:3.9.2
 org.apiguardian:apiguardian-api:1.1.2
-org.asynchttpclient:async-http-client-netty-utils:2.12.3
-org.asynchttpclient:async-http-client:2.12.3
+org.asynchttpclient:async-http-client-netty-utils:3.0.0
+org.asynchttpclient:async-http-client:3.0.0
 org.codehaus.groovy:groovy-all:2.4.21
 org.conscrypt:conscrypt-openjdk-uber:2.5.2
 org.eclipse.jetty.websocket:websocket-api:9.4.54.v20240208
@@ -503,7 +503,7 @@ org.jetbrains.kotlin:kotlin-stdlib-common:1.9.24
 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.24
 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.24
 org.jetbrains.kotlin:kotlin-stdlib:1.9.24
-org.jetbrains:annotations:17.0
+org.jetbrains:annotations:17.0.0
 org.locationtech.proj4j:proj4j:1.2.2
 org.lz4:lz4-java:1.8.0
 org.objenesis:objenesis:2.1
@@ -583,7 +583,6 @@ net.sf.jopt-simple:jopt-simple:5.0.4
 net.sourceforge.argparse4j:argparse4j:0.7.0
 org.checkerframework:checker-qual:3.44.0
 org.codehaus.mojo:animal-sniffer-annotations:1.23
-org.projectlombok:lombok:1.18.30
 org.reactivestreams:reactive-streams:1.0.4
 org.slf4j:slf4j-api:2.0.13
 org.slf4j:slf4j-reload4j:1.7.36
@@ -628,7 +627,7 @@ Common Development and Distribution License (CDDL) 1.0
 (see licenses/LICENSE-cddl-1.0.txt)
 
 com.sun.activation:javax.activation:1.2.0
-org.glassfish.jersey.containers:jersey-container-servlet-core:2.39
+org.glassfish.jersey.containers:jersey-container-servlet-core:2.42
 
 
 Common Development and Distribution License (CDDL) 1.1
@@ -637,7 +636,7 @@ Common Development and Distribution License (CDDL) 1.1
 
 com.github.pjfanning:jersey-json:1.20
 com.sun.xml.bind:jaxb-impl:2.2.3-1
-javax.activation:activation-api:1.2.0
+javax.activation:javax.activation-api:1.2.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.servlet:javax.servlet-api:4.0.1
 javax.xml.bind:jaxb-api:2.3.1

README.md

@@ -87,6 +87,15 @@ SELECT sum(clicks), sum(impressions) FROM AdAnalyticsTable
        daysSinceEpoch TOP 100
 ```
 
+## Contributing to Pinot
+Please refer to the [Contribution Guide](https://docs.pinot.apache.org/developers/developers-and-contributors/contribution-guidelines) for 
+more information on how to contribute to Apache Pinot.
+
+Normal Pinot builds are done using the `mvn clean install` command. 
+However this command can take a long time to run.
+For faster builds it is recommended to use `mvn verify -Ppinot-fastdev`, which disables some plugins that are not 
+actually needed for development. 
+
 ## Building Pinot
 More detailed instructions can be found at [Quick Demo](https://docs.pinot.apache.org/basics/getting-started/quick-start) section in the documentation.
 ```
@@ -95,7 +104,9 @@ $ git clone https://github.com/apache/pinot.git
 $ cd pinot
 
 # Build Pinot
-$ mvn clean install -DskipTests -Pbin-dist
+# -Pbin-dist is required to build the binary distribution
+# -Pbuild-shaded-jar is required to build the shaded jar, which is necessary for some features like spark connectors
+$ mvn clean install -DskipTests -Pbin-dist -Pbuild-shaded-jar
 
 # Run the Quick Demo
 $ cd build/

doap_Pinot.rdf

@@ -42,9 +42,9 @@ It's perfect for user-facing real-time analytics and other analytical use cases,
     <category rdf:resource="https://projects.apache.org/category/big-data" />
     <release>
       <Version>
-        <name>1.1.0</name>
-        <created>2024-03-24</created>
-        <revision>1.1.0</revision>
+        <name>1.2.0</name>
+        <created>2024-08-20</created>
+        <revision>1.2.0</revision>
       </Version>
     </release>
     <repository>

docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile

@@ -20,12 +20,19 @@
 ARG JAVA_VERSION=11
 ARG JDK_IMAGE=mcr.microsoft.com/openjdk/jdk
 
-FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu AS builder
+
+FROM ubuntu:24.10
+ARG JAVA_VERSION
+ENV LANG=en_US.UTF-8
+ENV JAVA_HOME="/usr/lib/jvm/msopenjdk-${JAVA_VERSION}"
+ENV PATH="${JAVA_HOME}/bin:${PATH}"
+COPY --from=builder $JAVA_HOME $JAVA_HOME
 
 LABEL [email protected]
 
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends vim less wget curl git python sysstat procps linux-tools-generic libtasn1-6 zstd && \
+  apt-get install -y --no-install-recommends vim less wget curl git python-is-python3 sysstat procps linux-tools-generic libtasn1-6 zstd ca-certificates && \
   rm -rf /var/lib/apt/lists/*
 
 RUN case `uname -m` in \

pinot-broker/pom.xml

@@ -25,7 +25,6 @@
     <artifactId>pinot</artifactId>
     <groupId>org.apache.pinot</groupId>
     <version>1.3.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
   </parent>
   <artifactId>pinot-broker</artifactId>
   <name>Pinot Broker</name>
@@ -36,34 +35,27 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.pinot</groupId>
-      <artifactId>pinot-core</artifactId>
+      <artifactId>pinot-query-runtime</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.pinot</groupId>
-      <artifactId>pinot-query-planner</artifactId>
+      <artifactId>pinot-timeseries-spi</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.pinot</groupId>
-      <artifactId>pinot-query-runtime</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-databind</artifactId>
+      <artifactId>pinot-timeseries-planner</artifactId>
     </dependency>
 
     <!-- Test -->
     <dependency>
       <groupId>org.apache.pinot</groupId>
       <artifactId>pinot-common</artifactId>
-      <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.pinot</groupId>
       <artifactId>pinot-core</artifactId>
-      <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
@@ -75,7 +67,6 @@
     <dependency>
       <groupId>org.apache.pinot</groupId>
       <artifactId>pinot-controller</artifactId>
-      <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>

pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java

@@ -47,6 +47,7 @@
 import javax.ws.rs.core.Response;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pinot.broker.broker.AccessControlFactory;
+import org.apache.pinot.broker.queryquota.QueryQuotaManager;
 import org.apache.pinot.broker.routing.BrokerRoutingManager;
 import org.apache.pinot.common.request.BrokerRequest;
 import org.apache.pinot.common.utils.DatabaseUtils;
@@ -92,6 +93,9 @@ public class PinotBrokerDebug {
   @Inject
   private ServerRoutingStatsManager _serverRoutingStatsManager;
 
+  @Inject
+  private QueryQuotaManager _queryQuotaManager;
+
   @Inject
   AccessControlFactory _accessControlFactory;
 
@@ -295,4 +299,28 @@ public Collection<? extends QueryResourceTracker> getQueryUsage() {
     ThreadResourceUsageAccountant threadAccountant = Tracing.getThreadAccountant();
     return threadAccountant.getQueryResources().values();
   }
+
+  @GET
+  @Path("debug/tables/queryQuota/{tableName}")
+  @Produces(MediaType.TEXT_PLAIN)
+  @Authorize(targetType = TargetType.TABLE, paramName = "tableName", action = Actions.Table.GET_TABLE_QUERY_QUOTA)
+  @ApiOperation(value = "Get the active query quota being imposed on the table", notes = "This is a debug endpoint, "
+      + "and won't maintain backward compatibility")
+  public String getTableQueryQuota(
+      @ApiParam(value = "Name of the table with type") @PathParam("tableName") String tableName,
+      @Context HttpHeaders headers) {
+    tableName = DatabaseUtils.translateTableName(tableName, headers);
+    return String.valueOf(_queryQuotaManager.getTableQueryQuota(tableName));
+  }
+
+  @GET
+  @Path("debug/databases/queryQuota/{databaseName}")
+  @Produces(MediaType.TEXT_PLAIN)
+  @Authorize(targetType = TargetType.CLUSTER, action = Actions.Cluster.GET_DATABASE_QUERY_QUOTA)
+  @ApiOperation(value = "Get the active query quota being imposed on the database", notes = "This is a debug endpoint, "
+      + "and won't maintain backward compatibility")
+  public String getDatabaseQueryQuota(
+      @ApiParam(value = "Name of the database") @PathParam("databaseName") String databaseName) {
+    return String.valueOf(_queryQuotaManager.getDatabaseQueryQuota(databaseName));
+  }
 }