diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 16a67a7ccf7..e509748c732 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,17 +1,10 @@ version: 2 -registries: - # Helps find updates for non Maven Central dependencies' - duckdb-snapshots: - type: maven-repository - url: https://oss.sonatype.org/content/repositories/snapshots/ updates: - package-ecosystem: "gradle" directory: "/" schedule: interval: "daily" - registries: - - duckdb-snapshots # Automatically update these dependencies allow: - - dependency-name: "org.duckdb:duckdb_jdbc" \ No newline at end of file + - dependency-name: "org.duckdb:duckdb_jdbc" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index cc12f88ffdc..8795493938c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,7 +4,7 @@ Add a list of affected components in the PR title in the following format: Valid component tags are: [da-vinci] (or [dvc]), [server], [controller], [router], [samza], [vpj], [fast-client] (or [fc]), [thin-client] (or [tc]), [changelog] (or [cc]), -[pulsar-sink], [producer], [admin-tool], [test], [build], [doc], [script], [compat] +[pulsar-sink], [producer], [admin-tool], [test], [build], [doc], [script], [compat], [protocol] Example title: [server][da-vinci] Use dedicated thread to persist data to storage engine diff --git a/.github/workflows/limit-pr-files-changed.yml b/.github/workflows/limit-pr-files-changed.yml new file mode 100644 index 00000000000..f22f59ad960 --- /dev/null +++ b/.github/workflows/limit-pr-files-changed.yml @@ -0,0 +1,108 @@ +name: Enforce Max Lines Changed Per File + +on: + pull_request: + paths: + - '**/*.java' # Monitor changes to Java files + - '**/*.avsc' # Monitor changes to Avro schema files + - '**/*.proto' # Monitor changes to proto schema files + +jobs: + enforce-lines-changed: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch PR Title via GitHub API + id: pr_details + run: | + PR_NUMBER=${{ github.event.pull_request.number }} + PR_TITLE=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER" | jq -r '.title') + echo "pr_title=$PR_TITLE" >> $GITHUB_ENV + PR_DESCRIPTION=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER" | jq -r '.body') + { + echo "pr_description<> "$GITHUB_ENV" + + - name: Check for Override Keyword + id: check_override + run: | + # Define the keyword for override + OVERRIDE_KEYWORD="DIFFSIZEOVERRIDE" + + # Check PR title and body for the keyword + if printf "%s%s" "$pr_title$pr_description" | grep -iq "$OVERRIDE_KEYWORD"; then + echo "Override keyword found. Skipping validation." + echo "override=true" >> $GITHUB_OUTPUT + else + echo "override=false" >> $GITHUB_OUTPUT + fi + - name: Calculate Lines Changed Per File + if: ${{ steps.check_override.outputs.override != 'true' }} + id: lines_changed_per_file + run: | + # Define the maximum allowed lines changed per file + MAX_LINES=500 + MAX_TOTAL_LINES_ADDED=2000 + TOTAL_LINED_ADDED=0 + # Get the diff of the PR and process each file + EXCEEDED=false + JAVA_FILE=false + SCHEMA_FILE=false + while IFS=$'\t' read -r added removed file; do + # Skip test files + if [[ "$file" != *src/main* ]]; then + echo "Skipping file: $file" + continue + fi + + if [[ "$file" == *.java ]]; then + JAVA_FILE=true + else + SCHEMA_FILE=true + fi + + # Calculate total lines changed for the file + TOTAL=$((added + removed)) + TOTAL_LINED_ADDED=$((TOTAL_LINED_ADDED + added)) + + echo "File: $file, Lines Changed: $TOTAL" + + # Fail if there are both schema and Java file changes + if [[ "JAVA_FILE" == true && "SCHEMA_FILE" == true ]]; then + echo "The PR has both schema and Java code changes, please make a separate PR for schema changes." + echo "For schema file changes, please update build.gradle to update 'versionOverrides' inside compileAvro task to use fixed protocol version" + exit 1 + fi + + if [[ "$TOTAL" -gt "$MAX_LINES" && "$JAVA_FILE" == "true" ]]; then + echo "File $file exceeds the maximum allowed lines changed ($TOTAL > $MAX_LINES)" + EXCEEDED=true + fi + done < <(git diff --numstat origin/main | grep -E '\.(java|avsc|proto)$') + + # Fail if total line added exceeds max limit + if [ "$TOTAL_LINED_ADDED" -gt "$MAX_TOTAL_LINES_ADDED" ]; then + echo "Total added lines of all files exceed the maximum allowed lines ($TOTAL_LINED_ADDED > $MAX_TOTAL_LINES_ADDED)." + exit 1 + fi + + # Fail if total number of lines added exceeds max limit + if [ "$EXCEEDED" = true ]; then + echo "Above files exceed the maximum allowed lines changed ($MAX_LINES)." + exit 1 + fi + + - name: Notify + if: failure() + run: | + echo "One or more files in the PR exceed the maximum allowed lines changed. Please review and adjust your changes to your files." diff --git a/.gitignore b/.gitignore index 44b4c8bcd52..2e2fc8f14d0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,4 @@ Gemfile.lock .bundles_cache docs/vendor/ clients/da-vinci-client/classHash*.txt -integrations/venice-duckdb/classHash*.txt +integrations/venice-duckdb/classHash*.txt \ No newline at end of file diff --git a/all-modules/build.gradle b/all-modules/build.gradle index a18574f0aba..d8cab968c8c 100644 --- a/all-modules/build.gradle +++ b/all-modules/build.gradle @@ -2,16 +2,9 @@ // via transitive dependencies instead of enumerating all modules individually. dependencies { rootProject.subprojects.each { subproject -> - // Excluding venice-duckdb as it's experimental and causes build issues - if (subproject.path != project.path && subproject.subprojects.isEmpty() && subproject.path != ':integrations:venice-duckdb') { - implementation (project(subproject.path)) { - exclude group: 'org.duckdb' - exclude module: 'venice-duckdb' - } + if (subproject.path != project.path && subproject.subprojects.isEmpty()) { + implementation project(subproject.path) } } - implementation (project(path: ':internal:venice-test-common', configuration: 'integrationTestUtils')) { - exclude group: 'org.duckdb' - exclude module: 'venice-duckdb' - } + implementation project(path: ':internal:venice-test-common', configuration: 'integrationTestUtils') } diff --git a/build.gradle b/build.gradle index 7f7a7096fe8..2f31bc0e8ee 100644 --- a/build.gradle +++ b/build.gradle @@ -64,6 +64,7 @@ ext.libraries = [ apacheSparkAvro: "org.apache.spark:spark-avro_${scala}:${apacheSparkVersion}", apacheSparkCore: "org.apache.spark:spark-core_${scala}:${apacheSparkVersion}", apacheSparkSql: "org.apache.spark:spark-sql_${scala}:${apacheSparkVersion}", + asm: "org.ow2.asm:asm:9.7", avro: "org.apache.avro:avro:${avroVersion}", avroCompiler: "org.apache.avro:avro-compiler:${avroVersion}", avroMapred: "org.apache.avro:avro-mapred:${avroVersion}", @@ -82,7 +83,7 @@ ext.libraries = [ commonsLang: 'commons-lang:commons-lang:2.6', conscrypt: 'org.conscrypt:conscrypt-openjdk-uber:2.5.2', d2: "com.linkedin.pegasus:d2:${pegasusVersion}", - duckdbJdbc: "org.duckdb:duckdb_jdbc:1.2.0-20250130.011706-145", // TODO: Remove SNAPSHOT when the real release is published! + duckdbJdbc: "org.duckdb:duckdb_jdbc:1.2.0", failsafe: 'net.jodah:failsafe:2.4.0', fastUtil: 'it.unimi.dsi:fastutil:8.3.0', grpcNettyShaded: "io.grpc:grpc-netty-shaded:${grpcVersion}", @@ -291,7 +292,6 @@ subprojects { implementation libraries.grpcProtobuf implementation libraries.grpcServices implementation libraries.grpcStub - implementation 'org.ow2.asm:asm:9.7' compileOnly libraries.tomcatAnnotations } @@ -314,6 +314,8 @@ subprojects { doFirst { def versionOverrides = [ // project(':internal:venice-common').file('src/main/resources/avro/StoreVersionState/v5', PathValidation.DIRECTORY) + project(':internal:venice-common').file('src/main/resources/avro/PartitionState/v15', PathValidation.DIRECTORY), + project(':internal:venice-common').file('src/main/resources/avro/KafkaMessageEnvelope/v11', PathValidation.DIRECTORY) ] def schemaDirs = [sourceDir] diff --git a/clients/da-vinci-client/build.gradle b/clients/da-vinci-client/build.gradle index 0f6666c18ca..4262f091837 100644 --- a/clients/da-vinci-client/build.gradle +++ b/clients/da-vinci-client/build.gradle @@ -33,6 +33,7 @@ dependencies { implementation project(':clients:venice-thin-client') implementation libraries.avroUtilFastserde + implementation libraries.asm implementation libraries.caffeine implementation libraries.fastUtil implementation libraries.httpAsyncClient @@ -55,4 +56,4 @@ checkerFramework { checkers = ['org.checkerframework.checker.nullness.NullnessChecker'] skipCheckerFramework = true excludeTests = true -} \ No newline at end of file +} diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/DaVinciBackend.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/DaVinciBackend.java index 855007b9229..d9fb4ae76d4 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/DaVinciBackend.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/DaVinciBackend.java @@ -126,6 +126,12 @@ public DaVinciBackend( LOGGER.info("Creating Da Vinci backend with managed clients: {}", managedClients); try { VeniceServerConfig backendConfig = configLoader.getVeniceServerConfig(); + + if (backendConfig.isDatabaseChecksumVerificationEnabled() && recordTransformerConfig != null) { + // The checksum verification will fail because DVRT transforms the values + throw new VeniceException("DaVinciRecordTransformer cannot be used with database checksum verification."); + } + useDaVinciSpecificExecutionStatusForError = backendConfig.useDaVinciSpecificExecutionStatusForError(); writeBatchingPushStatus = backendConfig.getDaVinciPushStatusCheckIntervalInMs() >= 0; this.configLoader = configLoader; @@ -295,10 +301,6 @@ public DaVinciBackend( } if (backendConfig.isBlobTransferManagerEnabled()) { - if (recordTransformerConfig != null) { - throw new VeniceException("DaVinciRecordTransformer doesn't support blob transfer."); - } - aggVersionedBlobTransferStats = new AggVersionedBlobTransferStats(metricsRepository, storeRepository, configLoader.getVeniceServerConfig()); diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/blobtransfer/BlobTransferUtil.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/blobtransfer/BlobTransferUtil.java index f3058342e07..76db7bedbfa 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/blobtransfer/BlobTransferUtil.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/blobtransfer/BlobTransferUtil.java @@ -1,7 +1,5 @@ package com.linkedin.davinci.blobtransfer; -import static com.linkedin.venice.client.store.ClientFactory.getTransportClient; - import com.linkedin.davinci.blobtransfer.client.NettyFileTransferClient; import com.linkedin.davinci.blobtransfer.server.P2PBlobTransferService; import com.linkedin.davinci.stats.AggVersionedBlobTransferStats; @@ -9,8 +7,6 @@ import com.linkedin.davinci.storage.StorageMetadataService; import com.linkedin.venice.blobtransfer.DaVinciBlobFinder; import com.linkedin.venice.blobtransfer.ServerBlobFinder; -import com.linkedin.venice.client.store.AbstractAvroStoreClient; -import com.linkedin.venice.client.store.AvroGenericStoreClientImpl; import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.helix.HelixCustomizedViewOfflinePushRepository; import com.linkedin.venice.meta.ReadOnlyStoreRepository; @@ -24,40 +20,14 @@ public class BlobTransferUtil { /** * Get a P2P blob transfer manager for DaVinci Client and start it. - * @param p2pTransferPort, the port used by the P2P transfer server and client + * @param p2pTransferServerPort, the port used by the P2P transfer server + * @param p2pTransferClientPort, the port used by the P2P transfer client * @param baseDir, the base directory of the underlying storage * @param clientConfig, the client config to start up a transport client * @param storageMetadataService, the storage metadata service * @return the blob transfer manager * @throws Exception */ - public static BlobTransferManager getP2PBlobTransferManagerForDVCAndStart( - int p2pTransferPort, - String baseDir, - ClientConfig clientConfig, - StorageMetadataService storageMetadataService, - ReadOnlyStoreRepository readOnlyStoreRepository, - StorageEngineRepository storageEngineRepository, - int maxConcurrentSnapshotUser, - int snapshotRetentionTimeInMin, - int blobTransferMaxTimeoutInMin, - AggVersionedBlobTransferStats aggVersionedBlobTransferStats, - BlobTransferUtils.BlobTransferTableFormat transferSnapshotTableFormat) { - return getP2PBlobTransferManagerForDVCAndStart( - p2pTransferPort, - p2pTransferPort, - baseDir, - clientConfig, - storageMetadataService, - readOnlyStoreRepository, - storageEngineRepository, - maxConcurrentSnapshotUser, - snapshotRetentionTimeInMin, - blobTransferMaxTimeoutInMin, - aggVersionedBlobTransferStats, - transferSnapshotTableFormat); - } - public static BlobTransferManager getP2PBlobTransferManagerForDVCAndStart( int p2pTransferServerPort, int p2pTransferClientPort, @@ -79,12 +49,10 @@ public static BlobTransferManager getP2PBlobTransferManagerForDVCAndStart( maxConcurrentSnapshotUser, snapshotRetentionTimeInMin, transferSnapshotTableFormat); - AbstractAvroStoreClient storeClient = - new AvroGenericStoreClientImpl<>(getTransportClient(clientConfig), false, clientConfig); BlobTransferManager manager = new NettyP2PBlobTransferManager( new P2PBlobTransferService(p2pTransferServerPort, baseDir, blobTransferMaxTimeoutInMin, blobSnapshotManager), new NettyFileTransferClient(p2pTransferClientPort, baseDir, storageMetadataService), - new DaVinciBlobFinder(storeClient), + new DaVinciBlobFinder(clientConfig), baseDir, aggVersionedBlobTransferStats); manager.start(); diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/AvroGenericDaVinciClient.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/AvroGenericDaVinciClient.java index b081a3c462c..181e6c3cff4 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/AvroGenericDaVinciClient.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/AvroGenericDaVinciClient.java @@ -69,6 +69,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -700,6 +701,11 @@ private VeniceConfigLoader buildVeniceConfig() { kafkaBootstrapServers = backendConfig.getString(KAFKA_BOOTSTRAP_SERVERS); } + String recordTransformerOutputValueSchema = "null"; + if (daVinciConfig.isRecordTransformerEnabled()) { + recordTransformerOutputValueSchema = Objects.toString(recordTransformerConfig.getOutputValueSchema(), "null"); + } + VeniceProperties config = new PropertyBuilder().put(KAFKA_ADMIN_CLASS, ApacheKafkaAdminAdapter.class.getName()) .put(ROCKSDB_LEVEL0_FILE_NUM_COMPACTION_TRIGGER, 4) // RocksDB default config .put(ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER, 20) // RocksDB default config @@ -712,11 +718,7 @@ private VeniceConfigLoader buildVeniceConfig() { .put(KAFKA_BOOTSTRAP_SERVERS, kafkaBootstrapServers) .put(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, daVinciConfig.getStorageClass() == StorageClass.MEMORY_BACKED_BY_DISK) .put(INGESTION_USE_DA_VINCI_CLIENT, true) - .put( - RECORD_TRANSFORMER_VALUE_SCHEMA, - daVinciConfig.isRecordTransformerEnabled() - ? recordTransformerConfig.getOutputValueSchema().toString() - : "null") + .put(RECORD_TRANSFORMER_VALUE_SCHEMA, recordTransformerOutputValueSchema) .put(INGESTION_ISOLATION_CONFIG_PREFIX + "." + INGESTION_MEMORY_LIMIT, -1) // Explicitly disable memory limiter // in Isolated Process .put(backendConfig.toProperties()) diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/BlockingDaVinciRecordTransformer.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/BlockingDaVinciRecordTransformer.java index 754e1b199b4..809417ad00f 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/BlockingDaVinciRecordTransformer.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/BlockingDaVinciRecordTransformer.java @@ -1,6 +1,10 @@ package com.linkedin.davinci.client; +import com.linkedin.davinci.store.AbstractStorageEngine; import com.linkedin.venice.annotation.Experimental; +import com.linkedin.venice.compression.VeniceCompressor; +import com.linkedin.venice.kafka.protocol.state.PartitionState; +import com.linkedin.venice.serialization.avro.InternalAvroSpecificSerializer; import com.linkedin.venice.utils.lazy.Lazy; import java.io.IOException; import java.util.concurrent.CountDownLatch; @@ -25,8 +29,8 @@ public BlockingDaVinciRecordTransformer( Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci) { - super(recordTransformer.getStoreVersion(), keySchema, inputValueSchema, outputValueSchema, storeRecordsInDaVinci); + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(recordTransformer.getStoreVersion(), keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); this.recordTransformer = recordTransformer; } @@ -58,6 +62,19 @@ public void onEndVersionIngestion(int currentVersion) { this.recordTransformer.onEndVersionIngestion(currentVersion); } + public void internalOnRecovery( + AbstractStorageEngine storageEngine, + int partitionId, + InternalAvroSpecificSerializer partitionStateSerializer, + Lazy compressor) { + // Using a wrapper around onRecovery because when calculating the class hash it grabs the name of the current class + // that is invoking it. If we directly invoke onRecovery from this class, the class hash will be calculated based + // on the contents of BlockingDaVinciRecordTransformer, not the user's implementation of DVRT. + // We also can't override onRecovery like the other methods because this method is final and the implementation + // should never be overriden. + this.recordTransformer.onRecovery(storageEngine, partitionId, partitionStateSerializer, compressor); + } + @Override public void close() throws IOException { this.recordTransformer.close(); diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformer.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformer.java index c2572b6bdcb..71740cb1ba5 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformer.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformer.java @@ -4,6 +4,8 @@ import com.linkedin.venice.annotation.Experimental; import com.linkedin.venice.compression.VeniceCompressor; import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.kafka.protocol.state.PartitionState; +import com.linkedin.venice.serialization.avro.InternalAvroSpecificSerializer; import com.linkedin.venice.utils.lazy.Lazy; import java.io.Closeable; import java.io.IOException; @@ -41,6 +43,11 @@ public abstract class DaVinciRecordTransformer implements Closeable { */ private final boolean storeRecordsInDaVinci; + /** + * Boolean to determine if we should always bootstrap from the Version Topic. + */ + private final boolean alwaysBootstrapFromVersionTopic; + /** * The key schema, which is immutable inside DaVinciClient. Users can modify the key if they are storing records in an external storage engine, but this must be managed by the user. */ @@ -63,17 +70,17 @@ public abstract class DaVinciRecordTransformer implements Closeable { * @param keySchema the key schema, which is immutable inside DaVinciClient. Users can modify the key if they are storing records in an external storage engine, but this must be managed by the user * @param inputValueSchema the value schema before transformation * @param outputValueSchema the value schema after transformation - * @param storeRecordsInDaVinci set this to false if you intend to store records in a custom storage, - * and not in the Da Vinci Client + * @param recordTransformerConfig the config for the record transformer */ public DaVinciRecordTransformer( int storeVersion, Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci) { + DaVinciRecordTransformerConfig recordTransformerConfig) { this.storeVersion = storeVersion; - this.storeRecordsInDaVinci = storeRecordsInDaVinci; + this.storeRecordsInDaVinci = recordTransformerConfig.getStoreRecordsInDaVinci(); + this.alwaysBootstrapFromVersionTopic = recordTransformerConfig.getAlwaysBootstrapFromVersionTopic(); this.keySchema = keySchema; // ToDo: Make use of inputValueSchema to support reader/writer schemas this.inputValueSchema = inputValueSchema; @@ -212,9 +219,10 @@ public final int getClassHash() { */ public final void onRecovery( AbstractStorageEngine storageEngine, - Integer partition, + int partitionId, + InternalAvroSpecificSerializer partitionStateSerializer, Lazy compressor) { - recordTransformerUtility.onRecovery(storageEngine, partition, compressor); + recordTransformerUtility.onRecovery(storageEngine, partitionId, partitionStateSerializer, compressor); } /** @@ -224,6 +232,13 @@ public final boolean getStoreRecordsInDaVinci() { return storeRecordsInDaVinci; } + /** + * @return {@link #alwaysBootstrapFromVersionTopic} + */ + public final boolean getAlwaysBootstrapFromVersionTopic() { + return alwaysBootstrapFromVersionTopic; + } + /** * Returns the schema for the key used in {@link DaVinciClient}'s operations. * diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerConfig.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerConfig.java index 925aec2520c..34126bfbd57 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerConfig.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerConfig.java @@ -1,5 +1,7 @@ package com.linkedin.davinci.client; +import com.linkedin.venice.exceptions.VeniceException; +import java.util.Optional; import org.apache.avro.Schema; @@ -10,19 +12,22 @@ public class DaVinciRecordTransformerConfig { private final DaVinciRecordTransformerFunctionalInterface recordTransformerFunction; private final Class outputValueClass; private final Schema outputValueSchema; + private final boolean storeRecordsInDaVinci; + private final boolean alwaysBootstrapFromVersionTopic; - /** - * @param recordTransformerFunction the functional interface for creating a {@link DaVinciRecordTransformer} - * @param outputValueClass the class of the output value - * @param outputValueSchema the schema of the output value - */ - public DaVinciRecordTransformerConfig( - DaVinciRecordTransformerFunctionalInterface recordTransformerFunction, - Class outputValueClass, - Schema outputValueSchema) { - this.recordTransformerFunction = recordTransformerFunction; - this.outputValueClass = outputValueClass; - this.outputValueSchema = outputValueSchema; + public DaVinciRecordTransformerConfig(Builder builder) { + this.recordTransformerFunction = Optional.ofNullable(builder.recordTransformerFunction) + .orElseThrow(() -> new VeniceException("recordTransformerFunction cannot be null")); + + this.outputValueClass = builder.outputValueClass; + this.outputValueSchema = builder.outputValueSchema; + if ((this.outputValueClass != null && this.outputValueSchema == null) + || (this.outputValueClass == null && this.outputValueSchema != null)) { + throw new VeniceException("outputValueClass and outputValueSchema must be defined together"); + } + + this.storeRecordsInDaVinci = Optional.ofNullable(builder.storeRecordsInDaVinci).orElse(true); + this.alwaysBootstrapFromVersionTopic = Optional.ofNullable(builder.alwaysBootstrapFromVersionTopic).orElse(false); } /** @@ -45,4 +50,76 @@ public Class getOutputValueClass() { public Schema getOutputValueSchema() { return outputValueSchema; } + + /** + * @return {@link #storeRecordsInDaVinci} + */ + public boolean getStoreRecordsInDaVinci() { + return storeRecordsInDaVinci; + } + + /** + * @return {@link #alwaysBootstrapFromVersionTopic} + */ + public boolean getAlwaysBootstrapFromVersionTopic() { + return alwaysBootstrapFromVersionTopic; + } + + public static class Builder { + private DaVinciRecordTransformerFunctionalInterface recordTransformerFunction; + private Class outputValueClass; + private Schema outputValueSchema; + private Boolean storeRecordsInDaVinci; + private Boolean alwaysBootstrapFromVersionTopic; + + /** + * @param recordTransformerFunction the functional interface for creating a {@link DaVinciRecordTransformer} + */ + public Builder setRecordTransformerFunction(DaVinciRecordTransformerFunctionalInterface recordTransformerFunction) { + this.recordTransformerFunction = recordTransformerFunction; + return this; + } + + /** + * Set this if you modify the schema during transformation. Must be used in conjunction with {@link #setOutputValueSchema(Schema)} + * @param outputValueClass the class of the output value + */ + public Builder setOutputValueClass(Class outputValueClass) { + this.outputValueClass = outputValueClass; + return this; + } + + /** + * Set this if you modify the schema during transformation. Must be used in conjunction with {@link #setOutputValueClass(Class)} + * @param outputValueSchema the schema of the output value + */ + public Builder setOutputValueSchema(Schema outputValueSchema) { + this.outputValueSchema = outputValueSchema; + return this; + } + + /** + * @param storeRecordsInDaVinci set this to false if you intend to store records in a custom storage, + * and not in the Da Vinci Client. + * Default is true. + */ + public Builder setStoreRecordsInDaVinci(boolean storeRecordsInDaVinci) { + this.storeRecordsInDaVinci = storeRecordsInDaVinci; + return this; + } + + /** + * @param alwaysBootstrapFromVersionTopic set this to true if {@link #storeRecordsInDaVinci} is false, and you're + * storing records in memory without being backed by disk. + * Default is false. + */ + public Builder setAlwaysBootstrapFromVersionTopic(boolean alwaysBootstrapFromVersionTopic) { + this.alwaysBootstrapFromVersionTopic = alwaysBootstrapFromVersionTopic; + return this; + } + + public DaVinciRecordTransformerConfig build() { + return new DaVinciRecordTransformerConfig(this); + } + } } diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerFunctionalInterface.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerFunctionalInterface.java index e821d935ff2..ec9df93407d 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerFunctionalInterface.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerFunctionalInterface.java @@ -13,5 +13,6 @@ DaVinciRecordTransformer apply( Integer storeVersion, Schema keySchema, Schema inputValueSchema, - Schema outputValueSchema); + Schema outputValueSchema, + DaVinciRecordTransformerConfig recordTransformerConfig); } diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerUtility.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerUtility.java index 8732a633a5b..50974ca88a1 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerUtility.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerUtility.java @@ -3,17 +3,18 @@ import com.linkedin.davinci.store.AbstractStorageEngine; import com.linkedin.davinci.store.AbstractStorageIterator; import com.linkedin.venice.compression.VeniceCompressor; -import com.linkedin.venice.exceptions.VeniceException; +import com.linkedin.venice.kafka.protocol.state.PartitionState; +import com.linkedin.venice.offsets.OffsetRecord; +import com.linkedin.venice.serialization.avro.InternalAvroSpecificSerializer; import com.linkedin.venice.serializer.AvroGenericDeserializer; import com.linkedin.venice.serializer.AvroSerializer; import com.linkedin.venice.utils.lazy.Lazy; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Optional; import org.apache.avro.Schema; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** @@ -23,6 +24,7 @@ * @param the type of the output value */ public class DaVinciRecordTransformerUtility { + private static final Logger LOGGER = LogManager.getLogger(DaVinciRecordTransformerUtility.class); private final DaVinciRecordTransformer recordTransformer; private final AvroGenericDeserializer keyDeserializer; private final AvroGenericDeserializer outputValueDeserializer; @@ -80,26 +82,18 @@ public final ByteBuffer prependSchemaIdToHeader(ByteBuffer valueBytes, int schem /** * @return true if transformer logic has changed since the last time the class was loaded */ - public boolean hasTransformerLogicChanged(int classHash) { - try { - String classHashPath = String.format("./classHash-%d.txt", recordTransformer.getStoreVersion()); - File f = new File(classHashPath); - if (f.exists()) { - try (BufferedReader br = new BufferedReader(new FileReader(classHashPath))) { - int storedClassHash = Integer.parseInt(br.readLine()); - if (storedClassHash == classHash) { - return false; - } - } - } + public boolean hasTransformerLogicChanged(int currentClassHash, OffsetRecord offsetRecord) { + Integer persistedClassHash = offsetRecord.getRecordTransformerClassHash(); - try (FileWriter fw = new FileWriter(classHashPath)) { - fw.write(String.valueOf(classHash)); - } - return true; - } catch (IOException e) { - throw new VeniceException("Failed to check if transformation logic has changed", e); + if (persistedClassHash != null && persistedClassHash == currentClassHash) { + LOGGER.info( + "A change in transformer logic has been detected. Persisted class hash = {}. New class hash = {}", + persistedClassHash, + currentClassHash); + return false; } + LOGGER.info("Transformer logic hasn't changed. Class hash = {}", currentClassHash); + return true; } /** @@ -107,18 +101,29 @@ public boolean hasTransformerLogicChanged(int classHash) { */ public final void onRecovery( AbstractStorageEngine storageEngine, - Integer partition, + int partitionId, + InternalAvroSpecificSerializer partitionStateSerializer, Lazy compressor) { - // ToDo: Store class hash in RocksDB to support blob transfer int classHash = recordTransformer.getClassHash(); - boolean transformerLogicChanged = hasTransformerLogicChanged(classHash); + Optional optionalOffsetRecord = storageEngine.getPartitionOffset(partitionId); + OffsetRecord offsetRecord = optionalOffsetRecord.orElseGet(() -> new OffsetRecord(partitionStateSerializer)); + + boolean transformerLogicChanged = hasTransformerLogicChanged(classHash, offsetRecord); + + if (recordTransformer.getAlwaysBootstrapFromVersionTopic() || transformerLogicChanged) { + LOGGER.info("Bootstrapping directly from the VersionTopic for partition {}", partitionId); - if (!recordTransformer.getStoreRecordsInDaVinci() || transformerLogicChanged) { // Bootstrap from VT - storageEngine.clearPartitionOffset(partition); + storageEngine.clearPartitionOffset(partitionId); + + // Offset record is deleted, so create a new one and persist it + offsetRecord = new OffsetRecord(partitionStateSerializer); + offsetRecord.setRecordTransformerClassHash(classHash); + storageEngine.putPartitionOffset(partitionId, offsetRecord); } else { // Bootstrap from local storage - AbstractStorageIterator iterator = storageEngine.getIterator(partition); + LOGGER.info("Bootstrapping from local storage for partition {}", partitionId); + AbstractStorageIterator iterator = storageEngine.getIterator(partitionId); for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { byte[] keyBytes = iterator.key(); byte[] valueBytes = iterator.value(); diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/ingestion/main/MainIngestionStorageMetadataService.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/ingestion/main/MainIngestionStorageMetadataService.java index 20d45a52e1d..066440b232c 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/ingestion/main/MainIngestionStorageMetadataService.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/ingestion/main/MainIngestionStorageMetadataService.java @@ -76,9 +76,10 @@ public void stopInner() throws Exception { } @Override - public void computeStoreVersionState(String topicName, Function mapFunction) - throws VeniceException { - topicStoreVersionStateMap.compute(topicName, (key, previousStoreVersionState) -> { + public StoreVersionState computeStoreVersionState( + String topicName, + Function mapFunction) throws VeniceException { + return topicStoreVersionStateMap.compute(topicName, (key, previousStoreVersionState) -> { StoreVersionState newStoreVersionState = mapFunction.apply(previousStoreVersionState); storeVersionStateSyncer.accept(topicName, newStoreVersionState); diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/kafka/consumer/SharedKafkaConsumer.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/kafka/consumer/SharedKafkaConsumer.java index 090b7816785..743bb4659a6 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/kafka/consumer/SharedKafkaConsumer.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/kafka/consumer/SharedKafkaConsumer.java @@ -3,12 +3,14 @@ import static com.linkedin.venice.utils.LatencyUtils.getElapsedTimeFromMsToMs; import com.linkedin.davinci.stats.AggKafkaConsumerServiceStats; +import com.linkedin.venice.annotation.UnderDevelopment; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.kafka.protocol.KafkaMessageEnvelope; import com.linkedin.venice.message.KafkaKey; import com.linkedin.venice.pubsub.PubSubTopicPartitionInfo; import com.linkedin.venice.pubsub.api.PubSubConsumerAdapter; import com.linkedin.venice.pubsub.api.PubSubMessage; +import com.linkedin.venice.pubsub.api.PubSubPosition; import com.linkedin.venice.pubsub.api.PubSubTopic; import com.linkedin.venice.pubsub.api.PubSubTopicPartition; import com.linkedin.venice.pubsub.api.exceptions.PubSubUnsubscribedTopicPartitionException; @@ -133,6 +135,13 @@ public synchronized void subscribe(PubSubTopicPartition pubSubTopicPartition, lo this.getClass().getSimpleName() + " does not support subscribe without specifying a version-topic."); } + @UnderDevelopment(value = "This API may not be implemented in all PubSubConsumerAdapter implementations.") + @Override + public synchronized void subscribe(PubSubTopicPartition pubSubTopicPartition, PubSubPosition lastReadPubSubPosition) { + throw new VeniceException( + this.getClass().getSimpleName() + " does not support subscribe without specifying a version-topic."); + } + synchronized void subscribe( PubSubTopic versionTopic, PubSubTopicPartition topicPartitionToSubscribe, diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTask.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTask.java index eafbfdd29c1..462f67535e4 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTask.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTask.java @@ -322,7 +322,7 @@ public abstract class StoreIngestionTask implements Runnable, Closeable { private final Schema recordTransformerInputValueSchema; private final AvroGenericDeserializer recordTransformerKeyDeserializer; private final SparseConcurrentList recordTransformerDeserializersByPutSchemaId; - private DaVinciRecordTransformer recordTransformer; + private BlockingDaVinciRecordTransformer recordTransformer; protected final String localKafkaServer; protected final int localKafkaClusterId; @@ -481,15 +481,25 @@ public StoreIngestionTask( this.recordTransformerInputValueSchema = schemaRepository.getSupersetOrLatestValueSchema(storeName).getSchema(); Schema outputValueSchema = recordTransformerConfig.getOutputValueSchema(); + // User doesn't intend on transforming records. Use input value schema instead. + if (outputValueSchema == null) { + outputValueSchema = this.recordTransformerInputValueSchema; + } + DaVinciRecordTransformer clientRecordTransformer = recordTransformerConfig.getRecordTransformerFunction() - .apply(versionNumber, keySchema, this.recordTransformerInputValueSchema, outputValueSchema); + .apply( + versionNumber, + keySchema, + this.recordTransformerInputValueSchema, + outputValueSchema, + recordTransformerConfig); this.recordTransformer = new BlockingDaVinciRecordTransformer( clientRecordTransformer, keySchema, this.recordTransformerInputValueSchema, outputValueSchema, - clientRecordTransformer.getStoreRecordsInDaVinci()); + recordTransformerConfig); this.recordTransformerDeserializersByPutSchemaId = new SparseConcurrentList<>(); versionedIngestionStats.registerTransformerLatencySensor(storeName, versionNumber); @@ -649,7 +659,7 @@ public synchronized void subscribePartition(PubSubTopicPartition topicPartition, int partitionNumber = topicPartition.getPartitionNumber(); if (recordTransformer != null) { - recordTransformer.onRecovery(storageEngine, partitionNumber, compressor); + recordTransformer.internalOnRecovery(storageEngine, partitionNumber, partitionStateSerializer, compressor); } partitionToPendingConsumerActionCountMap.computeIfAbsent(partitionNumber, x -> new AtomicInteger(0)) @@ -2948,47 +2958,72 @@ private void processStartOfPush( } else { sorted = startOfPush.sorted; } - beginBatchWrite(partition, sorted, partitionConsumptionState); - partitionConsumptionState.setStartOfPushTimestamp(startOfPushKME.producerMetadata.messageTimestamp); - ingestionNotificationDispatcher.reportStarted(partitionConsumptionState); - storageMetadataService.computeStoreVersionState(kafkaVersionTopic, previousStoreVersionState -> { - if (previousStoreVersionState == null) { - // No other partition of the same topic has started yet, let's initialize the StoreVersionState - StoreVersionState newStoreVersionState = new StoreVersionState(); - newStoreVersionState.sorted = sorted; - newStoreVersionState.chunked = startOfPush.chunked; - newStoreVersionState.compressionStrategy = startOfPush.compressionStrategy; - newStoreVersionState.compressionDictionary = startOfPush.compressionDictionary; - if (startOfPush.compressionStrategy == CompressionStrategy.ZSTD_WITH_DICT.getValue()) { - if (startOfPush.compressionDictionary == null) { + StoreVersionState persistedStoreVersionState = + storageMetadataService.computeStoreVersionState(kafkaVersionTopic, previousStoreVersionState -> { + if (previousStoreVersionState == null) { + // No other partition of the same topic has started yet, let's initialize the StoreVersionState + StoreVersionState newStoreVersionState = new StoreVersionState(); + newStoreVersionState.sorted = sorted; + newStoreVersionState.chunked = startOfPush.chunked; + newStoreVersionState.compressionStrategy = startOfPush.compressionStrategy; + newStoreVersionState.compressionDictionary = startOfPush.compressionDictionary; + if (startOfPush.compressionStrategy == CompressionStrategy.ZSTD_WITH_DICT.getValue()) { + if (startOfPush.compressionDictionary == null) { + throw new VeniceException( + "compression Dictionary should not be empty if CompressionStrategy is ZSTD_WITH_DICT"); + } + } + newStoreVersionState.batchConflictResolutionPolicy = startOfPush.timestampPolicy; + newStoreVersionState.startOfPushTimestamp = startOfPushKME.producerMetadata.messageTimestamp; + + LOGGER.info( + "Persisted {} for the first time following a SOP for topic {} with sorted: {}.", + StoreVersionState.class.getSimpleName(), + kafkaVersionTopic, + newStoreVersionState.sorted); + return newStoreVersionState; + } else if (previousStoreVersionState.chunked != startOfPush.chunked) { + // Something very wrong is going on ): ... throw new VeniceException( - "compression Dictionary should not be empty if CompressionStrategy is ZSTD_WITH_DICT"); + "Unexpected: received multiple " + ControlMessageType.START_OF_PUSH.name() + + " control messages with inconsistent 'chunked' fields within the same topic!"); + } else if (previousStoreVersionState.sorted != sorted) { + if (!isHybridMode()) { + // Something very wrong is going on ): ... + throw new VeniceException( + "Unexpected: received multiple " + ControlMessageType.START_OF_PUSH.name() + + " control messages with inconsistent 'sorted' fields within the same topic!" + + " And persisted sorted value: " + previousStoreVersionState.sorted + + " is different from the current one: " + sorted); + } + /** + * Because of the blob-db integration, `SIT` will forcibly set `sorted` to `false` for hybrid stores (check the + * above javadoc) and inconsistent `sorted` can happen during the rolling out/rolling back blob-db features. + * Here is one case how it can happen during the rolling out of blob-db: + * 1. P1 processes `SOP` with `sorted=true` and persist it in `StoreVersionState`. + * 2. P2 hasn't started processing `SOP` yet. + * 3. Restart the cluster and roll out blob-db feature. + * 4. when P2 processes `SOP` with `sorted=true`, it will forcibly set `sorted=false`, and it will be different + * from the previously persisted `StoreVersionState`. + * 5. This is fine as long as we just follow the previously persisted `StoreVersionState`. + * 6. Here are the reasons why step 5 is true: + * a. If the input for a given partition is truly sorted, it can always be ingested as unsorted data. + * b. If the input for a given partition is not sorted, the underlying `SSTFileWriter` will throw exception + * when we try to ingest it as sorted data. + */ + LOGGER.warn( + "Store version state for topic {} has already been initialized with a different value of 'sorted': {}", + kafkaVersionTopic, + previousStoreVersionState.sorted); } - } - newStoreVersionState.batchConflictResolutionPolicy = startOfPush.timestampPolicy; - newStoreVersionState.startOfPushTimestamp = startOfPushKME.producerMetadata.messageTimestamp; + // No need to mutate it, so we return it as is + return previousStoreVersionState; + }); - LOGGER.info( - "Persisted {} for the first time following a SOP for topic {}.", - StoreVersionState.class.getSimpleName(), - kafkaVersionTopic); - return newStoreVersionState; - } else if (previousStoreVersionState.sorted != sorted) { - // Something very wrong is going on ): ... - throw new VeniceException( - "Unexpected: received multiple " + ControlMessageType.START_OF_PUSH.name() - + " control messages with inconsistent 'sorted' fields within the same topic!"); - } else if (previousStoreVersionState.chunked != startOfPush.chunked) { - // Something very wrong is going on ): ... - throw new VeniceException( - "Unexpected: received multiple " + ControlMessageType.START_OF_PUSH.name() - + " control messages with inconsistent 'chunked' fields within the same topic!"); - } else { - // No need to mutate it, so we return it as is - return previousStoreVersionState; - } - }); + ingestionNotificationDispatcher.reportStarted(partitionConsumptionState); + beginBatchWrite(partition, persistedStoreVersionState.sorted, partitionConsumptionState); + partitionConsumptionState.setStartOfPushTimestamp(startOfPushKME.producerMetadata.messageTimestamp); } protected void processEndOfPush( diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/StorageEngineMetadataService.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/StorageEngineMetadataService.java index 6c8974f7481..dc8ef699ad0 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/StorageEngineMetadataService.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/StorageEngineMetadataService.java @@ -62,13 +62,15 @@ public void stopInner() throws Exception { } @Override - public void computeStoreVersionState(String topicName, Function mapFunction) - throws VeniceException { + public StoreVersionState computeStoreVersionState( + String topicName, + Function mapFunction) throws VeniceException { AbstractStorageEngine engine = getStorageEngineOrThrow(topicName); synchronized (engine) { StoreVersionState previousSVS = engine.getStoreVersionState(); StoreVersionState newSVS = mapFunction.apply(previousSVS); engine.putStoreVersionState(newSVS); + return newSVS; } } diff --git a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/StorageMetadataService.java b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/StorageMetadataService.java index 369733faf82..619eb0cec24 100644 --- a/clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/StorageMetadataService.java +++ b/clients/da-vinci-client/src/main/java/com/linkedin/davinci/storage/StorageMetadataService.java @@ -11,8 +11,9 @@ * This is a superset of the OffsetManager APIs, which also provide functions for storing store-version level state. */ public interface StorageMetadataService extends OffsetManager { - void computeStoreVersionState(String topicName, Function mapFunction) - throws VeniceException; + StoreVersionState computeStoreVersionState( + String topicName, + Function mapFunction) throws VeniceException; /** * This will clear all metadata, including store-version state and partition states, tied to {@param topicName}. diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java index f38915ee329..1ae8873fad0 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/client/AvroGenericDaVinciClientTest.java @@ -60,15 +60,11 @@ public AvroGenericDaVinciClient setUpClientWithRecordTransformer( daVinciConfig = new DaVinciConfig(); } - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new TestStringRecordTransformer( - storeVersion, - keySchema, - inputValueSchema, - outputValueSchema, - true), - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .setOutputValueClass(String.class) + .setOutputValueSchema(Schema.create(Schema.Type.STRING)) + .build(); daVinciConfig.setRecordTransformerConfig(recordTransformerConfig); VeniceProperties backendConfig = mock(VeniceProperties.class); diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/DaVinciConfigTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/DaVinciConfigTest.java index 1b156c24a61..ae8c53dfeed 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/DaVinciConfigTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/config/DaVinciConfigTest.java @@ -8,6 +8,7 @@ import com.linkedin.davinci.client.DaVinciConfig; import com.linkedin.davinci.client.DaVinciRecordTransformer; import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; +import com.linkedin.davinci.client.DaVinciRecordTransformerFunctionalInterface; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.venice.utils.lazy.Lazy; import java.io.IOException; @@ -22,8 +23,8 @@ public TestRecordTransformer( Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci) { - super(storeVersion, keySchema, inputValueSchema, outputValueSchema, storeRecordsInDaVinci); + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); } @Override @@ -47,16 +48,21 @@ public void testRecordTransformerEnabled() { DaVinciConfig config = new DaVinciConfig(); assertFalse(config.isRecordTransformerEnabled()); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new TestRecordTransformer( + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestRecordTransformer::new).build(); + + DaVinciRecordTransformerFunctionalInterface recordTransformerFunction = + (storeVersion, keySchema, inputValueSchema, outputValueSchema, config1) -> new TestRecordTransformer( storeVersion, keySchema, inputValueSchema, outputValueSchema, - true), - String.class, - Schema.create(Schema.Type.INT)); + dummyRecordTransformerConfig); + + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(recordTransformerFunction).build(); config.setRecordTransformerConfig(recordTransformerConfig); + assertTrue(config.isRecordTransformerEnabled()); } @@ -65,16 +71,21 @@ public void testGetAndSetRecordTransformer() { DaVinciConfig config = new DaVinciConfig(); assertNull(config.getRecordTransformerConfig()); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new TestRecordTransformer( + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestRecordTransformer::new).build(); + + DaVinciRecordTransformerFunctionalInterface recordTransformerFunction = + (storeVersion, keySchema, inputValueSchema, outputValueSchema, config1) -> new TestRecordTransformer( storeVersion, keySchema, inputValueSchema, outputValueSchema, - true), - String.class, - Schema.create(Schema.Type.INT)); + dummyRecordTransformerConfig); + + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(recordTransformerFunction).build(); config.setRecordTransformerConfig(recordTransformerConfig); + assertNotNull(config.getRecordTransformerConfig()); } diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java index 44c41a28735..b9b5265ec41 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/kafka/consumer/StoreIngestionTaskTest.java @@ -259,6 +259,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; +import org.mockito.stubbing.Answer; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; @@ -1305,6 +1306,7 @@ void setStoreVersionStateSupplier(StoreVersionState svs) { setupMockAbstractStorageEngine(metadataPartition); doReturn(svs).when(mockAbstractStorageEngine).getStoreVersionState(); doReturn(svs).when(mockStorageMetadataService).getStoreVersionState(topic); + doReturn(svs).when(mockStorageMetadataService).computeStoreVersionState(eq(topic), any()); } void setStoreVersionStateSupplier(boolean sorted) { @@ -2495,8 +2497,23 @@ public void testVeniceMessagesProcessingWithSortedInput(AAConfig aaConfig) throw }, aaConfig); } - @Test(dataProvider = "True-and-False", dataProviderClass = DataProviderUtils.class) - public void testVeniceMessagesProcessingWithSortedInputWithBlobMode(boolean blobMode) throws Exception { + @Test(dataProvider = "Boolean-and-Optional-Boolean", dataProviderClass = DataProviderUtils.class) + public void testVeniceMessagesProcessingWithSortedInputWithBlobMode(boolean blobMode, Boolean sortedFlagInSVS) + throws Exception { + if (sortedFlagInSVS != null) { + setStoreVersionStateSupplier(sortedFlagInSVS); + } else { + doReturn(null).when(mockStorageMetadataService).getStoreVersionState(any()); + } + doAnswer((Answer) invocationOnMock -> { + String topicName = invocationOnMock.getArgument(0, String.class); + Function mapFunction = invocationOnMock.getArgument(1, Function.class); + StoreVersionState updatedStoreVersionState = + mapFunction.apply(mockStorageMetadataService.getStoreVersionState(topicName)); + doReturn(updatedStoreVersionState).when(mockStorageMetadataService).getStoreVersionState(any()); + return updatedStoreVersionState; + }).when(mockStorageMetadataService).computeStoreVersionState(anyString(), any()); + localVeniceWriter.broadcastStartOfPush(true, new HashMap<>()); PubSubProduceResult putMetadata = (PubSubProduceResult) localVeniceWriter.put(putKeyFoo, putValue, EXISTING_SCHEMA_ID).get(); @@ -2518,7 +2535,13 @@ public void testVeniceMessagesProcessingWithSortedInputWithBlobMode(boolean blob .put(topic, PARTITION_FOO, getOffsetRecord(putMetadata.getOffset() - 1)); // Check database mode switches from deferred-write to transactional after EOP control message StoragePartitionConfig deferredWritePartitionConfig = new StoragePartitionConfig(topic, PARTITION_FOO); - deferredWritePartitionConfig.setDeferredWrite(!blobMode); + boolean deferredWrite; + if (!blobMode) { + deferredWrite = sortedFlagInSVS != null ? sortedFlagInSVS : true; + } else { + deferredWrite = sortedFlagInSVS != null ? sortedFlagInSVS : false; + } + deferredWritePartitionConfig.setDeferredWrite(deferredWrite); verify(mockAbstractStorageEngine, times(1)) .beginBatchWrite(eq(deferredWritePartitionConfig), any(), eq(Optional.empty())); StoragePartitionConfig transactionalPartitionConfig = new StoragePartitionConfig(topic, PARTITION_FOO); @@ -4785,7 +4808,7 @@ public void testMaybeSendIngestionHeartbeatWithHBSuccessOrFailure() throws Inter } @Test(dataProvider = "aaConfigProvider") - public void testStoreIngestionRecordTransformer(AAConfig aaConfig) throws Exception { + public void testSITRecordTransformer(AAConfig aaConfig) throws Exception { KafkaKey kafkaKey = new KafkaKey(MessageType.PUT, putKeyFoo); KafkaMessageEnvelope kafkaMessageEnvelope = new KafkaMessageEnvelope(); kafkaMessageEnvelope.messageType = MessageType.PUT.getValue(); @@ -4838,16 +4861,79 @@ public void testStoreIngestionRecordTransformer(AAConfig aaConfig) throws Except } }, aaConfig); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new TestStringRecordTransformer( - storeVersion, - keySchema, - inputValueSchema, - outputValueSchema, - true), - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .setOutputValueClass(String.class) + .setOutputValueSchema(Schema.create(Schema.Type.STRING)) + .build(); config.setRecordTransformerConfig(recordTransformerConfig); + + runTest(config); + + // Transformer error should never be recorded + verify(mockVersionedStorageIngestionStats, never()) + .recordTransformerError(eq(storeNameWithoutVersionInfo), anyInt(), anyDouble(), anyLong()); + } + + @Test(dataProvider = "aaConfigProvider") + public void testSITRecordTransformerUndefinedOutputValueClassAndSchema(AAConfig aaConfig) throws Exception { + KafkaKey kafkaKey = new KafkaKey(MessageType.PUT, putKeyFoo); + KafkaMessageEnvelope kafkaMessageEnvelope = new KafkaMessageEnvelope(); + kafkaMessageEnvelope.messageType = MessageType.PUT.getValue(); + Put put = new Put(); + + put.putValue = ByteBuffer.wrap(putValue); + put.replicationMetadataPayload = ByteBuffer.allocate(10); + kafkaMessageEnvelope.payloadUnion = put; + kafkaMessageEnvelope.producerMetadata = new ProducerMetadata(); + PubSubMessage pubSubMessage = new ImmutablePubSubMessage( + kafkaKey, + kafkaMessageEnvelope, + new PubSubTopicPartitionImpl(pubSubTopic, PARTITION_FOO), + 0, + 0, + 0); + + LeaderProducedRecordContext leaderProducedRecordContext = mock(LeaderProducedRecordContext.class); + when(leaderProducedRecordContext.getMessageType()).thenReturn(MessageType.PUT); + when(leaderProducedRecordContext.getValueUnion()).thenReturn(put); + when(leaderProducedRecordContext.getKeyBytes()).thenReturn(putKeyFoo); + + Schema myKeySchema = Schema.create(Schema.Type.INT); + SchemaEntry keySchemaEntry = mock(SchemaEntry.class); + when(keySchemaEntry.getSchema()).thenReturn(myKeySchema); + when(mockSchemaRepo.getKeySchema(storeNameWithoutVersionInfo)).thenReturn(keySchemaEntry); + + Schema myValueSchema = Schema.create(Schema.Type.STRING); + SchemaEntry valueSchemaEntry = mock(SchemaEntry.class); + when(valueSchemaEntry.getSchema()).thenReturn(myValueSchema); + when(mockSchemaRepo.getValueSchema(eq(storeNameWithoutVersionInfo), anyInt())).thenReturn(valueSchemaEntry); + when(mockSchemaRepo.getSupersetOrLatestValueSchema(eq(storeNameWithoutVersionInfo))).thenReturn(valueSchemaEntry); + + StoreIngestionTaskTestConfig config = new StoreIngestionTaskTestConfig(Collections.singleton(PARTITION_FOO), () -> { + TestUtils.waitForNonDeterministicAssertion( + 5, + TimeUnit.SECONDS, + () -> assertTrue(storeIngestionTaskUnderTest.hasAnySubscription())); + + try { + storeIngestionTaskUnderTest.produceToStoreBufferService( + pubSubMessage, + leaderProducedRecordContext, + PARTITION_FOO, + localKafkaConsumerService.kafkaUrl, + System.nanoTime(), + System.currentTimeMillis()); + } catch (InterruptedException e) { + throw new VeniceException(e); + } + }, aaConfig); + + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); + config.setRecordTransformerConfig(recordTransformerConfig); + runTest(config); // Transformer error should never be recorded @@ -4857,7 +4943,7 @@ public void testStoreIngestionRecordTransformer(AAConfig aaConfig) throws Except // Test to throw type error when performing record transformation with incompatible types @Test(dataProvider = "aaConfigProvider") - public void testStoreIngestionRecordTransformerError(AAConfig aaConfig) throws Exception { + public void testSITRecordTransformerError(AAConfig aaConfig) throws Exception { byte[] keyBytes = new byte[1]; KafkaKey kafkaKey = new KafkaKey(MessageType.PUT, keyBytes); KafkaMessageEnvelope kafkaMessageEnvelope = new KafkaMessageEnvelope(); @@ -4914,15 +5000,9 @@ public void testStoreIngestionRecordTransformerError(AAConfig aaConfig) throws E .recordTransformerError(eq(storeNameWithoutVersionInfo), anyInt(), anyDouble(), anyLong()); }, aaConfig); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new TestStringRecordTransformer( - storeVersion, - keySchema, - inputValueSchema, - outputValueSchema, - true), - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); config.setRecordTransformerConfig(recordTransformerConfig); runTest(config); } diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/DaVinciRecordTransformerConfigTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/DaVinciRecordTransformerConfigTest.java new file mode 100644 index 00000000000..70615c27914 --- /dev/null +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/DaVinciRecordTransformerConfigTest.java @@ -0,0 +1,89 @@ +package com.linkedin.davinci.transformer; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; +import com.linkedin.venice.exceptions.VeniceException; +import org.apache.avro.Schema; +import org.testng.annotations.Test; + + +public class DaVinciRecordTransformerConfigTest { + @Test + public void testRecordTransformerFunctionRequired() { + assertThrows(VeniceException.class, () -> new DaVinciRecordTransformerConfig.Builder().build()); + } + + @Test + public void testOutputValueClassAndSchemaBothRequired() { + assertThrows( + VeniceException.class, + () -> new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction(TestStringRecordTransformer::new) + .setOutputValueClass(String.class) + .build()); + + assertThrows( + VeniceException.class, + () -> new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction(TestStringRecordTransformer::new) + .setOutputValueSchema(Schema.create(Schema.Type.STRING)) + .build()); + } + + @Test + public void testDefaults() { + Class outputValueClass = String.class; + Schema outputValueSchema = Schema.create(Schema.Type.STRING); + + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .setOutputValueClass(outputValueClass) + .setOutputValueSchema(outputValueSchema) + .build(); + + assertNotNull(recordTransformerConfig.getRecordTransformerFunction()); + assertEquals(recordTransformerConfig.getOutputValueClass(), outputValueClass); + assertEquals(recordTransformerConfig.getOutputValueSchema(), outputValueSchema); + assertTrue(recordTransformerConfig.getStoreRecordsInDaVinci()); + assertFalse(recordTransformerConfig.getAlwaysBootstrapFromVersionTopic()); + } + + @Test + public void testNonDefaults() { + Class outputValueClass = String.class; + Schema outputValueSchema = Schema.create(Schema.Type.STRING); + + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .setOutputValueClass(outputValueClass) + .setOutputValueSchema(outputValueSchema) + .setStoreRecordsInDaVinci(false) + .setAlwaysBootstrapFromVersionTopic(true) + .build(); + + assertNotNull(recordTransformerConfig.getRecordTransformerFunction()); + assertEquals(recordTransformerConfig.getOutputValueClass(), outputValueClass); + assertEquals(recordTransformerConfig.getOutputValueSchema(), outputValueSchema); + assertFalse(recordTransformerConfig.getStoreRecordsInDaVinci()); + assertTrue(recordTransformerConfig.getAlwaysBootstrapFromVersionTopic()); + } + + @Test + public void testUndefinedOutputValueClassAndSchema() { + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); + + assertNotNull(recordTransformerConfig.getRecordTransformerFunction()); + assertNull(recordTransformerConfig.getOutputValueClass()); + assertNull(recordTransformerConfig.getOutputValueSchema()); + assertTrue(recordTransformerConfig.getStoreRecordsInDaVinci()); + assertFalse(recordTransformerConfig.getAlwaysBootstrapFromVersionTopic()); + } +} diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/RecordTransformerTest.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/RecordTransformerTest.java index 0645f2be9ba..5be3f7b611f 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/RecordTransformerTest.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/RecordTransformerTest.java @@ -13,38 +13,44 @@ import com.linkedin.davinci.client.BlockingDaVinciRecordTransformer; import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.davinci.client.DaVinciRecordTransformerUtility; import com.linkedin.davinci.store.AbstractStorageEngine; import com.linkedin.davinci.store.AbstractStorageIterator; import com.linkedin.venice.compression.VeniceCompressor; +import com.linkedin.venice.kafka.protocol.state.PartitionState; +import com.linkedin.venice.offsets.OffsetRecord; +import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; +import com.linkedin.venice.serialization.avro.InternalAvroSpecificSerializer; import com.linkedin.venice.utils.lazy.Lazy; -import java.io.File; +import java.util.Optional; import org.apache.avro.Schema; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; public class RecordTransformerTest { static final int storeVersion = 1; - - @BeforeMethod - @AfterClass - public void deleteClassHash() { - File file = new File(String.format("./classHash-%d.txt", storeVersion)); - if (file.exists()) { - assertTrue(file.delete()); - } - } + static final int partitionId = 0; + static final InternalAvroSpecificSerializer partitionStateSerializer = + AvroProtocolDefinition.PARTITION_STATE.getSerializer(); @Test public void testRecordTransformer() { Schema keySchema = Schema.create(Schema.Type.INT); Schema valueSchema = Schema.create(Schema.Type.STRING); - DaVinciRecordTransformer recordTransformer = - new TestStringRecordTransformer(storeVersion, keySchema, valueSchema, valueSchema, false); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .setStoreRecordsInDaVinci(false) + .build(); + + DaVinciRecordTransformer recordTransformer = new TestStringRecordTransformer( + storeVersion, + keySchema, + valueSchema, + valueSchema, + dummyRecordTransformerConfig); assertEquals(recordTransformer.getStoreVersion(), storeVersion); assertEquals(recordTransformer.getKeySchema().getType(), Schema.Type.INT); @@ -66,8 +72,13 @@ public void testRecordTransformer() { DaVinciRecordTransformerUtility recordTransformerUtility = recordTransformer.getRecordTransformerUtility(); - assertTrue(recordTransformerUtility.hasTransformerLogicChanged(classHash)); - assertFalse(recordTransformerUtility.hasTransformerLogicChanged(classHash)); + OffsetRecord offsetRecord = new OffsetRecord(partitionStateSerializer); + + assertTrue(recordTransformerUtility.hasTransformerLogicChanged(classHash, offsetRecord)); + + offsetRecord.setRecordTransformerClassHash(classHash); + + assertFalse(recordTransformerUtility.hasTransformerLogicChanged(classHash, offsetRecord)); } @Test @@ -75,8 +86,16 @@ public void testOnRecovery() { Schema keySchema = Schema.create(Schema.Type.INT); Schema valueSchema = Schema.create(Schema.Type.STRING); - DaVinciRecordTransformer recordTransformer = - new TestStringRecordTransformer(storeVersion, keySchema, valueSchema, valueSchema, true); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); + + DaVinciRecordTransformer recordTransformer = new TestStringRecordTransformer( + storeVersion, + keySchema, + valueSchema, + valueSchema, + dummyRecordTransformerConfig); assertEquals(recordTransformer.getStoreVersion(), storeVersion); AbstractStorageIterator iterator = mock(AbstractStorageIterator.class); @@ -87,18 +106,71 @@ public void testOnRecovery() { AbstractStorageEngine storageEngine = mock(AbstractStorageEngine.class); Lazy compressor = Lazy.of(() -> mock(VeniceCompressor.class)); - int partitionNumber = 1; - recordTransformer.onRecovery(storageEngine, partitionNumber, compressor); - verify(storageEngine, times(1)).clearPartitionOffset(partitionNumber); + OffsetRecord offsetRecord = new OffsetRecord(partitionStateSerializer); + when(storageEngine.getPartitionOffset(partitionId)).thenReturn(Optional.of(offsetRecord)); + + recordTransformer.onRecovery(storageEngine, partitionId, partitionStateSerializer, compressor); + verify(storageEngine, times(1)).clearPartitionOffset(partitionId); + + // Reset the mock to clear previous interactions + reset(storageEngine); + + offsetRecord.setRecordTransformerClassHash(recordTransformer.getClassHash()); + assertEquals((int) offsetRecord.getRecordTransformerClassHash(), recordTransformer.getClassHash()); + + // class hash should be the same when the OffsetRecord is serialized then deserialized + byte[] offsetRecordBytes = offsetRecord.toBytes(); + OffsetRecord deserializedOffsetRecord = new OffsetRecord(offsetRecordBytes, partitionStateSerializer); + assertEquals((int) deserializedOffsetRecord.getRecordTransformerClassHash(), recordTransformer.getClassHash()); + + when(storageEngine.getPartitionOffset(partitionId)).thenReturn(Optional.of(offsetRecord)); + + // Execute the onRecovery method again to test the case where the classHash exists + when(storageEngine.getIterator(partitionId)).thenReturn(iterator); + recordTransformer.onRecovery(storageEngine, partitionId, partitionStateSerializer, compressor); + verify(storageEngine, never()).clearPartitionOffset(partitionId); + verify(storageEngine, times(1)).getIterator(partitionId); + } + + @Test + public void testOnRecoveryAlwaysBootstrapFromVersionTopic() { + Schema keySchema = Schema.create(Schema.Type.INT); + Schema valueSchema = Schema.create(Schema.Type.STRING); + + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .setAlwaysBootstrapFromVersionTopic(true) + .build(); + + DaVinciRecordTransformer recordTransformer = new TestStringRecordTransformer( + storeVersion, + keySchema, + valueSchema, + valueSchema, + dummyRecordTransformerConfig); + assertEquals(recordTransformer.getStoreVersion(), storeVersion); + + AbstractStorageEngine storageEngine = mock(AbstractStorageEngine.class); + Lazy compressor = Lazy.of(() -> mock(VeniceCompressor.class)); + + OffsetRecord offsetRecord = new OffsetRecord(partitionStateSerializer); + when(storageEngine.getPartitionOffset(partitionId)).thenReturn(Optional.of(offsetRecord)); + + recordTransformer.onRecovery(storageEngine, partitionId, partitionStateSerializer, compressor); + verify(storageEngine, times(1)).clearPartitionOffset(partitionId); // Reset the mock to clear previous interactions reset(storageEngine); - // Execute the onRecovery method again to test the case where the classHash file exists - when(storageEngine.getIterator(partitionNumber)).thenReturn(iterator); - recordTransformer.onRecovery(storageEngine, partitionNumber, compressor); - verify(storageEngine, never()).clearPartitionOffset(partitionNumber); - verify(storageEngine, times(1)).getIterator(partitionNumber); + offsetRecord.setRecordTransformerClassHash(recordTransformer.getClassHash()); + assertEquals((int) offsetRecord.getRecordTransformerClassHash(), recordTransformer.getClassHash()); + + when(storageEngine.getPartitionOffset(partitionId)).thenReturn(Optional.of(offsetRecord)); + + // Execute the onRecovery method again to test the case where the classHash exists + recordTransformer.onRecovery(storageEngine, partitionId, partitionStateSerializer, compressor); + verify(storageEngine, times(1)).clearPartitionOffset(partitionId); + verify(storageEngine, never()).getIterator(partitionId); } @Test @@ -106,8 +178,16 @@ public void testBlockingRecordTransformer() { Schema keySchema = Schema.create(Schema.Type.INT); Schema valueSchema = Schema.create(Schema.Type.STRING); - DaVinciRecordTransformer recordTransformer = - new TestStringRecordTransformer(storeVersion, keySchema, valueSchema, valueSchema, true); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); + + DaVinciRecordTransformer recordTransformer = new TestStringRecordTransformer( + storeVersion, + keySchema, + valueSchema, + valueSchema, + dummyRecordTransformerConfig); assertEquals(recordTransformer.getStoreVersion(), storeVersion); recordTransformer = new BlockingDaVinciRecordTransformer<>( @@ -115,7 +195,7 @@ public void testBlockingRecordTransformer() { keySchema, valueSchema, valueSchema, - recordTransformer.getStoreRecordsInDaVinci()); + dummyRecordTransformerConfig); recordTransformer.onStartVersionIngestion(true); assertTrue(recordTransformer.getStoreRecordsInDaVinci()); @@ -134,5 +214,4 @@ public void testBlockingRecordTransformer() { recordTransformer.onEndVersionIngestion(2); } - } diff --git a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/TestStringRecordTransformer.java b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/TestStringRecordTransformer.java index f2d83aec1c6..c12c558629d 100644 --- a/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/TestStringRecordTransformer.java +++ b/clients/da-vinci-client/src/test/java/com/linkedin/davinci/transformer/TestStringRecordTransformer.java @@ -1,6 +1,7 @@ package com.linkedin.davinci.transformer; import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.venice.utils.lazy.Lazy; import java.io.IOException; @@ -14,8 +15,8 @@ public TestStringRecordTransformer( Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci) { - super(storeVersion, keySchema, inputValueSchema, outputValueSchema, storeRecordsInDaVinci); + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); } @Override diff --git a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java index 828579f7792..aa3ce1857d2 100644 --- a/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java +++ b/clients/venice-admin-tool/src/main/java/com/linkedin/venice/AdminTool.java @@ -841,8 +841,7 @@ private static void createNewStore(CommandLine cmd) throws Exception { String valueSchemaFile = getRequiredArgument(cmd, Arg.VALUE_SCHEMA, Command.NEW_STORE); String valueSchema = readFile(valueSchemaFile); String owner = getOptionalArgument(cmd, Arg.OWNER, ""); - boolean isVsonStore = - Utils.parseBooleanFromString(getOptionalArgument(cmd, Arg.VSON_STORE, "false"), "isVsonStore"); + boolean isVsonStore = Utils.parseBooleanOrThrow(getOptionalArgument(cmd, Arg.VSON_STORE, "false"), "isVsonStore"); if (isVsonStore) { keySchema = VsonAvroSchemaAdapter.parse(keySchema).toString(); valueSchema = VsonAvroSchemaAdapter.parse(valueSchema).toString(); @@ -1126,7 +1125,7 @@ private static void longParam(CommandLine cmd, Arg param, Consumer setter, } private static void booleanParam(CommandLine cmd, Arg param, Consumer setter, Set argSet) { - genericParam(cmd, param, s -> Utils.parseBooleanFromString(s, param.toString()), setter, argSet); + genericParam(cmd, param, s -> Utils.parseBooleanOrThrow(s, param.toString()), setter, argSet); } private static void stringMapParam( @@ -2536,8 +2535,7 @@ private static void createNewStoreWithAcl(CommandLine cmd) throws Exception { String valueSchema = readFile(valueSchemaFile); String aclPerms = getRequiredArgument(cmd, Arg.ACL_PERMS, Command.NEW_STORE); String owner = getOptionalArgument(cmd, Arg.OWNER, ""); - boolean isVsonStore = - Utils.parseBooleanFromString(getOptionalArgument(cmd, Arg.VSON_STORE, "false"), "isVsonStore"); + boolean isVsonStore = Utils.parseBooleanOrThrow(getOptionalArgument(cmd, Arg.VSON_STORE, "false"), "isVsonStore"); if (isVsonStore) { keySchema = VsonAvroSchemaAdapter.parse(keySchema).toString(); valueSchema = VsonAvroSchemaAdapter.parse(valueSchema).toString(); diff --git a/clients/venice-client/src/main/java/com/linkedin/venice/fastclient/ClientConfig.java b/clients/venice-client/src/main/java/com/linkedin/venice/fastclient/ClientConfig.java index ae044255859..106b9556a0a 100644 --- a/clients/venice-client/src/main/java/com/linkedin/venice/fastclient/ClientConfig.java +++ b/clients/venice-client/src/main/java/com/linkedin/venice/fastclient/ClientConfig.java @@ -87,6 +87,7 @@ public class ClientConfig { private boolean projectionFieldValidation; private Set harClusters; private final InstanceHealthMonitor instanceHealthMonitor; + private final boolean retryBudgetEnabled; private final MetricsRepository metricsRepository; @@ -121,7 +122,8 @@ private ClientConfig( boolean projectionFieldValidation, long longTailRetryBudgetEnforcementWindowInMs, Set harClusters, - InstanceHealthMonitor instanceHealthMonitor) { + InstanceHealthMonitor instanceHealthMonitor, + boolean retryBudgetEnabled) { if (storeName == null || storeName.isEmpty()) { throw new VeniceClientException("storeName param shouldn't be empty"); } @@ -239,6 +241,7 @@ private ClientConfig( } else { this.instanceHealthMonitor = instanceHealthMonitor; } + this.retryBudgetEnabled = retryBudgetEnabled; } public String getStoreName() { @@ -375,6 +378,10 @@ public InstanceHealthMonitor getInstanceHealthMonitor() { return instanceHealthMonitor; } + public boolean isRetryBudgetEnabled() { + return retryBudgetEnabled; + } + public static class ClientConfigBuilder { private MetricsRepository metricsRepository; private String statsPrefix = ""; @@ -419,6 +426,8 @@ public static class ClientConfigBuilder { private InstanceHealthMonitor instanceHealthMonitor; + private boolean retryBudgetEnabled = true; + public ClientConfigBuilder setStoreName(String storeName) { this.storeName = storeName; return this; @@ -587,6 +596,11 @@ public ClientConfigBuilder setInstanceHealthMonitor(InstanceHealthMonit return this; } + public ClientConfigBuilder setRetryBudgetEnabled(boolean retryBudgetEnabled) { + this.retryBudgetEnabled = retryBudgetEnabled; + return this; + } + public ClientConfigBuilder clone() { return new ClientConfigBuilder().setStoreName(storeName) .setR2Client(r2Client) @@ -619,7 +633,8 @@ public ClientConfigBuilder clone() { .setProjectionFieldValidationEnabled(projectionFieldValidation) .setLongTailRetryBudgetEnforcementWindowInMs(longTailRetryBudgetEnforcementWindowInMs) .setHARClusters(harClusters) - .setInstanceHealthMonitor(instanceHealthMonitor); + .setInstanceHealthMonitor(instanceHealthMonitor) + .setRetryBudgetEnabled(retryBudgetEnabled); } public ClientConfig build() { @@ -654,7 +669,8 @@ public ClientConfig build() { projectionFieldValidation, longTailRetryBudgetEnforcementWindowInMs, harClusters, - instanceHealthMonitor); + instanceHealthMonitor, + retryBudgetEnabled); } } } diff --git a/clients/venice-client/src/main/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClient.java b/clients/venice-client/src/main/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClient.java index 650afe4bf28..635e8d25adc 100644 --- a/clients/venice-client/src/main/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClient.java +++ b/clients/venice-client/src/main/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClient.java @@ -54,8 +54,8 @@ public class RetriableAvroGenericStoreClient extends DelegatingAvroStoreCl private static final Logger LOGGER = LogManager.getLogger(RetriableAvroGenericStoreClient.class); // Default value of 0.1 meaning only 10 percent of the user requests are allowed to trigger long tail retry private static final double LONG_TAIL_RETRY_BUDGET_PERCENT_DECIMAL = 0.1d; - private static final String SINGLE_KEY_LONG_TAIL_RETRY_STATS_PREFIX = "single-key-long-tail-retry-manager-"; - private static final String MULTI_KEY_LONG_TAIL_RETRY_STATS_PREFIX = "multi-key-long-tail-retry-manager-"; + public static final String SINGLE_KEY_LONG_TAIL_RETRY_STATS_PREFIX = "single-key-long-tail-retry-manager-"; + public static final String MULTI_KEY_LONG_TAIL_RETRY_STATS_PREFIX = "multi-key-long-tail-retry-manager-"; public RetriableAvroGenericStoreClient( InternalAvroStoreClient delegate, @@ -76,7 +76,7 @@ public RetriableAvroGenericStoreClient( this.longTailRetryThresholdForComputeInMicroSeconds = clientConfig.getLongTailRetryThresholdForComputeInMicroSeconds(); this.timeoutProcessor = timeoutProcessor; - if (longTailRetryEnabledForSingleGet) { + if (longTailRetryEnabledForSingleGet && clientConfig.isRetryBudgetEnabled()) { this.singleKeyLongTailRetryManager = new RetryManager( clientConfig.getClusterStats().getMetricsRepository(), SINGLE_KEY_LONG_TAIL_RETRY_STATS_PREFIX + clientConfig.getStoreName(), @@ -84,7 +84,7 @@ public RetriableAvroGenericStoreClient( LONG_TAIL_RETRY_BUDGET_PERCENT_DECIMAL, retryManagerExecutorService); } - if (longTailRetryEnabledForBatchGet) { + if (longTailRetryEnabledForBatchGet && clientConfig.isRetryBudgetEnabled()) { this.multiKeyLongTailRetryManager = new RetryManager( clientConfig.getClusterStats().getMetricsRepository(), MULTI_KEY_LONG_TAIL_RETRY_STATS_PREFIX + clientConfig.getStoreName(), @@ -141,7 +141,9 @@ protected CompletableFuture get(GetRequestContext requestContext, K key) thro // if longTailRetry is not enabled for single get, simply return the original future return originalRequestFuture; } - singleKeyLongTailRetryManager.recordRequest(); + if (singleKeyLongTailRetryManager != null) { + singleKeyLongTailRetryManager.recordRequest(); + } final CompletableFuture retryFuture = new CompletableFuture<>(); final CompletableFuture finalFuture = new CompletableFuture<>(); @@ -153,7 +155,8 @@ protected CompletableFuture get(GetRequestContext requestContext, K key) thro retryFuture.completeExceptionally(savedException.get()); return; } - if (savedException.get() != null || singleKeyLongTailRetryManager.isRetryAllowed()) { + if (savedException.get() != null || singleKeyLongTailRetryManager == null + || singleKeyLongTailRetryManager.isRetryAllowed()) { super.get(requestContext, key).whenComplete((value, throwable) -> { if (throwable != null) { retryFuture.completeExceptionally(throwable); @@ -311,7 +314,8 @@ private , RESPONSE> void retryStreamingMu finalRequestCompletionFuture.completeExceptionally(throwable); return; } - if (throwable != null || multiKeyLongTailRetryManager.isRetryAllowed(pendingKeysFuture.keySet().size())) { + if (throwable != null || multiKeyLongTailRetryManager == null + || multiKeyLongTailRetryManager.isRetryAllowed(pendingKeysFuture.keySet().size())) { Set pendingKeys = Collections.unmodifiableSet(pendingKeysFuture.keySet()); R retryRequestContext = requestContextConstructor.construct(pendingKeys.size(), requestContext.isPartialSuccessAllowed); @@ -359,7 +363,9 @@ private , RESPONSE> void retryStreamingMu savedException, pendingKeysFuture, scheduledRetryTask)); - multiKeyLongTailRetryManager.recordRequests(requestContext.numKeysInRequest); + if (multiKeyLongTailRetryManager != null) { + multiKeyLongTailRetryManager.recordRequests(requestContext.numKeysInRequest); + } finalRequestCompletionFuture.whenComplete((ignore, finalException) -> { if (!scheduledRetryTask.isDone()) { diff --git a/clients/venice-client/src/test/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClientRetryBudgetTest.java b/clients/venice-client/src/test/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClientRetryBudgetTest.java new file mode 100644 index 00000000000..2a98ff2b8fe --- /dev/null +++ b/clients/venice-client/src/test/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClientRetryBudgetTest.java @@ -0,0 +1,9 @@ +package com.linkedin.venice.fastclient; + +public class RetriableAvroGenericStoreClientRetryBudgetTest extends RetriableAvroGenericStoreClientTest { + @Override + protected boolean isRetryBudgetEnabled() { + return true; + } + +} diff --git a/clients/venice-client/src/test/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClientTest.java b/clients/venice-client/src/test/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClientTest.java index f274f8d36de..8d0070fef67 100644 --- a/clients/venice-client/src/test/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClientTest.java +++ b/clients/venice-client/src/test/java/com/linkedin/venice/fastclient/RetriableAvroGenericStoreClientTest.java @@ -99,6 +99,10 @@ public Object[][] fcRequestTypesAndTwoBoolean() { .allPermutationGenerator(FASTCLIENT_REQUEST_TYPES, DataProviderUtils.BOOLEAN, DataProviderUtils.BOOLEAN); } + protected boolean isRetryBudgetEnabled() { + return false; + } + @BeforeClass public void setUp() { timeoutProcessor = new TimeoutProcessor(null, true, 1); @@ -113,6 +117,7 @@ public void setUp() { .setLongTailRetryThresholdForBatchGetInMicroSeconds( (int) TimeUnit.MILLISECONDS.toMicros(LONG_TAIL_RETRY_THRESHOLD_IN_MS)) .setLongTailRetryEnabledForCompute(true) + .setRetryBudgetEnabled(isRetryBudgetEnabled()) .setLongTailRetryThresholdForComputeInMicroSeconds( (int) TimeUnit.MILLISECONDS.toMicros(LONG_TAIL_RETRY_THRESHOLD_IN_MS)); BATCH_GET_KEYS.add("test_key_1"); @@ -546,8 +551,29 @@ private void validateMetrics( if (batchGet) { assertNotNull(batchGetRequestContext.retryContext.retryRequestContext); assertEquals(batchGetRequestContext.retryContext.retryRequestContext.numKeysInRequest, (int) expectedKeyCount); + + // Check retry budget metrics + String batchGetRetryBudgetMetricName = + "." + RetriableAvroGenericStoreClient.MULTI_KEY_LONG_TAIL_RETRY_STATS_PREFIX + clientConfig.getStoreName() + + "--retry_limit_per_seconds.Gauge"; + if (isRetryBudgetEnabled()) { + assertNotNull(metrics.get(batchGetRetryBudgetMetricName), "Retry limit per second metric should not be null"); + } else { + assertNull(metrics.get(batchGetRetryBudgetMetricName), "Retry limit per second metric should be null"); + } } else if (singleGet) { assertTrue(getRequestContext.retryContext.longTailRetryRequestTriggered); + // Check retry budget metrics + String singleGetRetryBudgetMetricName = + "." + RetriableAvroGenericStoreClient.SINGLE_KEY_LONG_TAIL_RETRY_STATS_PREFIX + clientConfig.getStoreName() + + "--retry_limit_per_seconds.Gauge"; + if (isRetryBudgetEnabled()) { + assertNotNull( + metrics.get(singleGetRetryBudgetMetricName), + "Retry limit per second metric should not be null"); + } else { + assertNull(metrics.get(singleGetRetryBudgetMetricName), "Retry limit per second metric should be null"); + } } } else { assertFalse(metrics.get(metricsPrefix + "long_tail_retry_request.OccurrenceRate").value() > 0); diff --git a/clients/venice-thin-client/src/main/java/com/linkedin/venice/client/store/AbstractAvroStoreClient.java b/clients/venice-thin-client/src/main/java/com/linkedin/venice/client/store/AbstractAvroStoreClient.java index 264955d6b70..4381520e9d1 100644 --- a/clients/venice-thin-client/src/main/java/com/linkedin/venice/client/store/AbstractAvroStoreClient.java +++ b/clients/venice-thin-client/src/main/java/com/linkedin/venice/client/store/AbstractAvroStoreClient.java @@ -75,6 +75,7 @@ public abstract class AbstractAvroStoreClient extends InternalAvroStoreCli private RecordDeserializer streamingFooterRecordDeserializer; private TransportClient transportClient; private final Executor deserializationExecutor; + private final Executor veniceClientWarmUpExecutor; private final CompressorFactory compressorFactory; private final String storageRequestPath; private final String computeRequestPath; @@ -132,6 +133,8 @@ protected AbstractAvroStoreClient( this.compressorFactory = new CompressorFactory(); this.storageRequestPath = TYPE_STORAGE + "/" + clientConfig.getStoreName(); this.computeRequestPath = TYPE_COMPUTE + "/" + clientConfig.getStoreName(); + this.veniceClientWarmUpExecutor = Executors + .newFixedThreadPool(1, new DaemonThreadFactory("Venice-Client-Warmup-For-" + clientConfig.getStoreName())); } @Override @@ -714,7 +717,7 @@ private void warmUpVeniceClient() { * If the D2 client isn't retry in the async warm-up phase, it will be delayed to the first query. * Essentially, this is a best-effort. */ - CompletableFuture.runAsync(this::getKeySerializerWithRetryWithLongInterval); + CompletableFuture.runAsync(this::getKeySerializerWithRetryWithLongInterval, veniceClientWarmUpExecutor); } } } diff --git a/clients/venice-thin-client/src/main/java/com/linkedin/venice/client/store/ClientFactory.java b/clients/venice-thin-client/src/main/java/com/linkedin/venice/client/store/ClientFactory.java index c1e9426d4e1..59b58885b87 100644 --- a/clients/venice-thin-client/src/main/java/com/linkedin/venice/client/store/ClientFactory.java +++ b/clients/venice-thin-client/src/main/java/com/linkedin/venice/client/store/ClientFactory.java @@ -22,7 +22,7 @@ public class ClientFactory { private static Function configToTransportClientProviderForTests = null; // Visible for testing - static void setUnitTestMode() { + public static void setUnitTestMode() { unitTestMode = true; } @@ -33,7 +33,7 @@ static void resetUnitTestMode() { // Allow for overriding with mock D2Client for unit tests. The caller must release the object to prevent side-effects // VisibleForTesting - static void setTransportClientProvider(Function transportClientProvider) { + public static void setTransportClientProvider(Function transportClientProvider) { if (!unitTestMode) { throw new VeniceUnsupportedOperationException("setTransportClientProvider in non-unit-test-mode"); } diff --git a/docs/dev_guide/how_to/recommended_development_workflow.md b/docs/dev_guide/how_to/recommended_development_workflow.md index e840a05d839..3fd6db85884 100644 --- a/docs/dev_guide/how_to/recommended_development_workflow.md +++ b/docs/dev_guide/how_to/recommended_development_workflow.md @@ -33,7 +33,7 @@ The GitHub issue should contain the detailed problem statement. 6. The PR title should usually be of the form `[component1]...[componentN]: Concise commit message`. * Valid tags are: `[da-vinci]` (or `[dvc]`), `[server]`, `[controller]`, `[router]`, `[samza]`, `[vpj]`, `[fast-client]` (or `[fc]`), `[thin-client]` (or `[tc]`), `[changelog]` (or `[cc]`), - `[producer]`, `[admin-tool]`, `[test]`, `[build]`, `[doc]`, `[script]`, `[compat]` + `[producer]`, `[admin-tool]`, `[test]`, `[build]`, `[doc]`, `[script]`, `[compat]`, `[protocol]` * `[compat]` tag means there are compatibility related changes in this PR, including upgrading protocol version, upgrading system store value schemas, etc. When there is a compatibility related change, it usually requires a specific deployment order, like upgrading controller before upgrading server. In this case, please explicitly call out the required deployment order in the commit message. 7. If the pull request is still a work in progress, and so is not ready to be merged, but needs to be pushed to GitHub to facilitate review, then create the PR as a [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) PR diff --git a/docs/user_guide/read_api/da_vinci_client.md b/docs/user_guide/read_api/da_vinci_client.md index 9d412810f13..cce7259c48a 100644 --- a/docs/user_guide/read_api/da_vinci_client.md +++ b/docs/user_guide/read_api/da_vinci_client.md @@ -12,44 +12,36 @@ cache. Future updates to the data continue to be streamed in and applied to the ## Record Transformer This feature enables applications to transform records before they're stored in the Da Vinci Client -or a custom storage of your choice. +or redirected to a custom storage of your choice. It's capable of handling records that are compressed and/or chunked. -### Usage +### Example Usage Steps to use the record transformer: 1. Implement the [DaVinciRecordTransformer](http://venicedb.org/javadoc/com/linkedin/davinci/client/DaVinciRecordTransformer.html) abstract class. 2. Create an instance of [DaVinciRecordTransformerConfig](http://venicedb.org/javadoc/com/linkedin/davinci/client/DaVinciRecordTransformerConfig.html). -3. Pass the instance of the config into [setRecordTransformerConfig()](https://venicedb.org/javadoc/com/linkedin/davinci/client/DaVinciConfig.html#setRecordTransformerConfig(com.linkedin.davinci.client.DaVinciRecordTransformerConfig)). - -When a message is being consumed, the -[DaVinciRecordTransformer](http://venicedb.org/javadoc/com/linkedin/davinci/client/DaVinciRecordTransformer.html) will -modify the value before it is written to storage. +3. Pass the instance of the config into [setRecordTransformerConfig()](https://venicedb.org/javadoc/com/linkedin/davinci/client/DaVinciConfig.html#setRecordTransformerConfig(com.linkedin.davinci.client.DaVinciRecordTransformerConfig)). Here's an example `DaVinciRecordTransformer` implementation: ``` -package com.linkedin.davinci.transformer; - import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; +import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.venice.utils.lazy.Lazy; +import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.util.Utf8; public class StringRecordTransformer extends DaVinciRecordTransformer { - public TestStringRecordTransformer(int storeVersion, boolean storeRecordsInDaVinci) { - super(storeVersion, storeRecordsInDaVinci); - } - - @Override - public Schema getKeySchema() { - return Schema.create(Schema.Type.INT); - } - - @Override - public Schema getOutputValueSchema() { - return Schema.create(Schema.Type.STRING); + public StringRecordTransformer( + int storeVersion, + Schema keySchema, + Schema inputValueSchema, + Schema outputValueSchema, + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); } @Override @@ -84,8 +76,63 @@ public class StringRecordTransformer extends DaVinciRecordTransformer new StringRecordTransformer(storeVersion, true), - String.class, Schema.create(Schema.Type.STRING)); -config.setRecordTransformerFunction((storeVersion) -> new StringRecordTransformer(storeVersion, true)); +DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction(StringRecordTransformer::new) + .build(); +config.setRecordTransformerConfig(recordTransformerConfig); +``` + +### Schema Modification +If you want to modify the Value Schema of the record, here's an example `DaVinciRecordTransformer` implementation: +``` +import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; +import com.linkedin.davinci.client.DaVinciRecordTransformerResult; +import com.linkedin.venice.utils.lazy.Lazy; +import java.io.IOException; +import org.apache.avro.Schema; + + +/** + * Transforms int values to strings + */ +public class IntToStringRecordTransformer extends DaVinciRecordTransformer { + public IntToStringRecordTransformer( + int storeVersion, + Schema keySchema, + Schema inputValueSchema, + Schema outputValueSchema, + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); + } + + @Override + public DaVinciRecordTransformerResult transform(Lazy key, Lazy value) { + String valueStr = value.get().toString(); + String transformedValue = valueStr + "Transformed"; + return new DaVinciRecordTransformerResult<>(DaVinciRecordTransformerResult.Result.TRANSFORMED, transformedValue); + } + + @Override + public void processPut(Lazy key, Lazy value) { + return; + } + + @Override + public void close() throws IOException { + + } +} +``` + +Here's an example `DaVinciRecordTransformerConfig` implementation: +``` +DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction(IntToStringRecordTransformer::new) + .setOutputValueClass(String.class) + .setOutputValueSchema(Schema.create(Schema.Type.STRING)) + .build(); +config.setRecordTransformerConfig(recordTransformerConfig); ``` \ No newline at end of file diff --git a/gradle/spotbugs/exclude.xml b/gradle/spotbugs/exclude.xml index b900862aac8..6418a827e36 100644 --- a/gradle/spotbugs/exclude.xml +++ b/gradle/spotbugs/exclude.xml @@ -78,6 +78,7 @@ + diff --git a/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java b/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java index b5f39c1267f..3c0c1ff37cb 100644 --- a/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java +++ b/integrations/venice-duckdb/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java @@ -29,6 +29,7 @@ import com.linkedin.davinci.client.DaVinciClient; import com.linkedin.davinci.client.DaVinciConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; +import com.linkedin.davinci.client.DaVinciRecordTransformerFunctionalInterface; import com.linkedin.davinci.client.factory.CachingDaVinciClientFactory; import com.linkedin.venice.D2.D2ClientUtils; import com.linkedin.venice.client.store.ClientConfig; @@ -70,7 +71,6 @@ import org.apache.logging.log4j.Logger; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -115,16 +115,6 @@ public void cleanUp() { Utils.closeQuietlyWithErrorLogged(this.cluster); } - @BeforeMethod - @AfterClass - public void deleteClassHash() { - int storeVersion = 1; - File file = new File(String.format("./classHash-%d.txt", storeVersion)); - if (file.exists()) { - assertTrue(file.delete()); - } - } - @Test(timeOut = TEST_TIMEOUT) public void testRecordTransformer() throws Exception { DaVinciConfig clientConfig = new DaVinciConfig(); @@ -147,18 +137,22 @@ public void testRecordTransformer() throws Exception { metricsRepository, backendConfig)) { Set columnsToProject = Collections.emptySet(); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new DuckDBDaVinciRecordTransformer( + + DaVinciRecordTransformerFunctionalInterface recordTransformerFunction = + (storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> new DuckDBDaVinciRecordTransformer( storeVersion, keySchema, inputValueSchema, outputValueSchema, - false, + config, tmpDir.getAbsolutePath(), storeName, - columnsToProject), - GenericRecord.class, - NAME_RECORD_V1_SCHEMA); + columnsToProject); + + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(recordTransformerFunction) + .setStoreRecordsInDaVinci(false) + .build(); clientConfig.setRecordTransformerConfig(recordTransformerConfig); DaVinciClient clientWithRecordTransformer = diff --git a/integrations/venice-duckdb/src/main/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformer.java b/integrations/venice-duckdb/src/main/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformer.java index 3d8f425936e..46ab0868f81 100644 --- a/integrations/venice-duckdb/src/main/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformer.java +++ b/integrations/venice-duckdb/src/main/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformer.java @@ -3,6 +3,7 @@ import static com.linkedin.venice.sql.AvroToSQL.UnsupportedTypeHandling.SKIP; import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.sql.AvroToSQL; @@ -44,11 +45,11 @@ public DuckDBDaVinciRecordTransformer( Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci, + DaVinciRecordTransformerConfig recordTransformerConfig, String baseDir, String storeNameWithoutVersionInfo, Set columnsToProject) { - super(storeVersion, keySchema, inputValueSchema, outputValueSchema, storeRecordsInDaVinci); + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); this.storeNameWithoutVersionInfo = storeNameWithoutVersionInfo; this.versionTableName = buildStoreNameWithVersion(storeVersion); this.duckDBUrl = "jdbc:duckdb:" + baseDir + "/" + duckDBFilePath; diff --git a/integrations/venice-duckdb/src/test/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformerTest.java b/integrations/venice-duckdb/src/test/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformerTest.java index 89711e63218..6ab0fbb8022 100644 --- a/integrations/venice-duckdb/src/test/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformerTest.java +++ b/integrations/venice-duckdb/src/test/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformerTest.java @@ -8,12 +8,15 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.davinci.client.DaVinciRecordTransformerUtility; +import com.linkedin.venice.kafka.protocol.state.PartitionState; +import com.linkedin.venice.offsets.OffsetRecord; +import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; +import com.linkedin.venice.serialization.avro.InternalAvroSpecificSerializer; import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.lazy.Lazy; -import java.io.File; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -24,35 +27,32 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; public class DuckDBDaVinciRecordTransformerTest { static final int storeVersion = 1; + static final int partitionId = 0; + static final InternalAvroSpecificSerializer partitionStateSerializer = + AvroProtocolDefinition.PARTITION_STATE.getSerializer(); static final String storeName = "test_store"; private final Set columnsToProject = Collections.emptySet(); - @BeforeMethod - @AfterClass - public void deleteClassHash() { - File file = new File(String.format("./classHash-%d.txt", storeVersion)); - if (file.exists()) { - assertTrue(file.delete()); - } - } - @Test - public void testRecordTransformer() throws IOException { + public void testRecordTransformer() { String tempDir = Utils.getTempDataDirectory().getAbsolutePath(); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction((storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> null) + .setStoreRecordsInDaVinci(false) + .build(); + try (DuckDBDaVinciRecordTransformer recordTransformer = new DuckDBDaVinciRecordTransformer( storeVersion, SINGLE_FIELD_RECORD_SCHEMA, NAME_RECORD_V1_SCHEMA, NAME_RECORD_V1_SCHEMA, - false, + dummyRecordTransformerConfig, tempDir, storeName, columnsToProject)) { @@ -90,8 +90,13 @@ public void testRecordTransformer() throws IOException { DaVinciRecordTransformerUtility recordTransformerUtility = recordTransformer.getRecordTransformerUtility(); - assertTrue(recordTransformerUtility.hasTransformerLogicChanged(classHash)); - assertFalse(recordTransformerUtility.hasTransformerLogicChanged(classHash)); + OffsetRecord offsetRecord = new OffsetRecord(partitionStateSerializer); + + assertTrue(recordTransformerUtility.hasTransformerLogicChanged(classHash, offsetRecord)); + + offsetRecord.setRecordTransformerClassHash(classHash); + + assertFalse(recordTransformerUtility.hasTransformerLogicChanged(classHash, offsetRecord)); } } @@ -99,12 +104,17 @@ public void testRecordTransformer() throws IOException { public void testVersionSwap() throws SQLException { String tempDir = Utils.getTempDataDirectory().getAbsolutePath(); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction((storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> null) + .setStoreRecordsInDaVinci(false) + .build(); + DuckDBDaVinciRecordTransformer recordTransformer_v1 = new DuckDBDaVinciRecordTransformer( 1, SINGLE_FIELD_RECORD_SCHEMA, NAME_RECORD_V1_SCHEMA, NAME_RECORD_V1_SCHEMA, - false, + dummyRecordTransformerConfig, tempDir, storeName, columnsToProject); @@ -113,7 +123,7 @@ public void testVersionSwap() throws SQLException { SINGLE_FIELD_RECORD_SCHEMA, NAME_RECORD_V1_SCHEMA, NAME_RECORD_V1_SCHEMA, - false, + dummyRecordTransformerConfig, tempDir, storeName, columnsToProject); @@ -162,12 +172,17 @@ public void testTwoTablesConcurrently() throws SQLException { String store1 = "store1"; String store2 = "store2"; + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction((storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> null) + .setStoreRecordsInDaVinci(false) + .build(); + DuckDBDaVinciRecordTransformer recordTransformerForStore1 = new DuckDBDaVinciRecordTransformer( 1, SINGLE_FIELD_RECORD_SCHEMA, NAME_RECORD_V1_SCHEMA, NAME_RECORD_V1_SCHEMA, - false, + dummyRecordTransformerConfig, tempDir, store1, columnsToProject); @@ -176,7 +191,7 @@ public void testTwoTablesConcurrently() throws SQLException { TWO_FIELDS_RECORD_SCHEMA, NAME_RECORD_V1_SCHEMA, NAME_RECORD_V1_SCHEMA, - false, + dummyRecordTransformerConfig, tempDir, store2, columnsToProject); diff --git a/internal/venice-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java b/internal/venice-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java index 49e7ed3a40f..e1ea241a424 100644 --- a/internal/venice-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java +++ b/internal/venice-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java @@ -13,6 +13,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -80,7 +81,16 @@ public VeniceClusterInitializer(String storeName, int routerPort) { Properties clusterConfig = new Properties(); clusterConfig.put(ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); clusterConfig.put(ConfigKeys.ROUTER_ENABLE_SSL, false); - this.veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0, 2, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + this.veniceCluster = ServiceFactory.getVeniceCluster(options); Properties serverProperties = new Properties(); serverProperties.put(ConfigKeys.SERVER_COMPUTE_FAST_AVRO_ENABLED, true); this.veniceCluster.addVeniceServer(new Properties(), serverProperties); diff --git a/internal/venice-client-common/src/main/java/com/linkedin/venice/exceptions/VeniceNoStoreException.java b/internal/venice-client-common/src/main/java/com/linkedin/venice/exceptions/VeniceNoStoreException.java index 85f78384732..7f7cc0019ec 100644 --- a/internal/venice-client-common/src/main/java/com/linkedin/venice/exceptions/VeniceNoStoreException.java +++ b/internal/venice-client-common/src/main/java/com/linkedin/venice/exceptions/VeniceNoStoreException.java @@ -9,6 +9,7 @@ public class VeniceNoStoreException extends VeniceException { private final String storeName; private final String clusterName; + public static final String DOES_NOT_EXISTS = " does not exist"; public VeniceNoStoreException(String storeName, String clusterName) { super(getErrorMessage(storeName, clusterName, null), ErrorType.STORE_NOT_FOUND); @@ -54,7 +55,7 @@ public int getHttpStatusCode() { } private static String getErrorMessage(String storeName, String clusterName, String additionalMessage) { - StringBuilder errorBuilder = new StringBuilder().append("Store: ").append(storeName).append(" does not exist"); + StringBuilder errorBuilder = new StringBuilder().append("Store: ").append(storeName).append(DOES_NOT_EXISTS); if (clusterName != null) { errorBuilder.append(" in cluster ").append(clusterName); } else { diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/annotation/UnderDevelopment.java b/internal/venice-common/src/main/java/com/linkedin/venice/annotation/UnderDevelopment.java new file mode 100644 index 00000000000..a6155819100 --- /dev/null +++ b/internal/venice-common/src/main/java/com/linkedin/venice/annotation/UnderDevelopment.java @@ -0,0 +1,20 @@ +package com.linkedin.venice.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * Indicates that an API (class, method, or field) is under development and is not yet stable. + * This annotation warns users that the annotated element may change or be removed in future versions. + */ +@Retention(RetentionPolicy.CLASS) // Retained in class files but not available at runtime. +@Target({ ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.TYPE, ElementType.FIELD }) +public @interface UnderDevelopment { + /** + * An optional message providing additional details about the development status. + */ + String value() default ""; +} diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/annotation/VisibleForTesting.java b/internal/venice-common/src/main/java/com/linkedin/venice/annotation/VisibleForTesting.java new file mode 100644 index 00000000000..acd84c497ab --- /dev/null +++ b/internal/venice-common/src/main/java/com/linkedin/venice/annotation/VisibleForTesting.java @@ -0,0 +1,18 @@ +package com.linkedin.venice.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * Indicates that a method, class, or field is more visible than otherwise necessary + * for the purpose of testing. This is useful for documenting code and communicating + * that a specific visibility level (e.g., `public` or `protected`) is intentional + * for testing purposes. + */ +@Retention(RetentionPolicy.CLASS) // Retained in class files but not available at runtime. +@Target({ ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.TYPE, ElementType.FIELD }) +public @interface VisibleForTesting { +} diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/blobtransfer/DaVinciBlobFinder.java b/internal/venice-common/src/main/java/com/linkedin/venice/blobtransfer/DaVinciBlobFinder.java index 18a3149ea94..10d6b6c0b80 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/blobtransfer/DaVinciBlobFinder.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/blobtransfer/DaVinciBlobFinder.java @@ -1,12 +1,16 @@ package com.linkedin.venice.blobtransfer; +import static com.linkedin.venice.client.store.ClientFactory.getTransportClient; import static com.linkedin.venice.controllerapi.ControllerApiConstants.NAME; import static com.linkedin.venice.controllerapi.ControllerApiConstants.STORE_PARTITION; import static com.linkedin.venice.controllerapi.ControllerApiConstants.STORE_VERSION; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.venice.client.store.AbstractAvroStoreClient; +import com.linkedin.venice.client.store.AvroGenericStoreClientImpl; +import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.utils.ObjectMapperFactory; +import com.linkedin.venice.utils.concurrent.VeniceConcurrentHashMap; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -29,14 +33,35 @@ public class DaVinciBlobFinder implements BlobFinder { private static final String TYPE_BLOB_DISCOVERY = "blob_discovery"; private static final String ERROR_DISCOVERY_MESSAGE = "Error finding DVC peers for blob transfer in store: %s, version: %d, partition: %d"; - private final AbstractAvroStoreClient storeClient; + private final ClientConfig clientConfig; + private VeniceConcurrentHashMap storeToClientMap; - public DaVinciBlobFinder(AbstractAvroStoreClient storeClient) { - this.storeClient = storeClient; + public DaVinciBlobFinder(ClientConfig clientConfig) { + this.storeToClientMap = new VeniceConcurrentHashMap<>(); + this.clientConfig = clientConfig; + } + + /** + * Get the store client for the given store name + * @param storeName + * @return the store client + */ + AbstractAvroStoreClient getStoreClient(String storeName) { + return storeToClientMap.computeIfAbsent(storeName, k -> { + // update the config with respective store name + ClientConfig storeClientConfig = ClientConfig.cloneConfig(clientConfig).setStoreName(storeName); + AbstractAvroStoreClient storeLevelClient = + new AvroGenericStoreClientImpl<>(getTransportClient(storeClientConfig), false, storeClientConfig); + storeLevelClient.start(); + LOGGER.info("Started store client for store: {}", storeName); + return storeLevelClient; + }); } @Override public BlobPeersDiscoveryResponse discoverBlobPeers(String storeName, int version, int partition) { + AbstractAvroStoreClient storeClient = getStoreClient(storeName); + String uri = buildUriForBlobDiscovery(storeName, version, partition); CompletableFuture futureResponse = CompletableFuture.supplyAsync(() -> { try { @@ -85,6 +110,8 @@ private BlobPeersDiscoveryResponse handleError( @Override public void close() throws IOException { - storeClient.close(); + for (AbstractAvroStoreClient storeClient: storeToClientMap.values()) { + storeClient.close(); + } } } diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/meta/RetryManager.java b/internal/venice-common/src/main/java/com/linkedin/venice/meta/RetryManager.java index aeb6f1b5e97..4674c87c9b5 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/meta/RetryManager.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/meta/RetryManager.java @@ -99,27 +99,34 @@ public boolean isRetryAllowed(int numberOfRetries) { } private void updateRetryTokenBucket() { - if (retryBudgetEnabled.get() && requestCount.get() > 0) { + if (retryBudgetEnabled.get()) { try { + /** + * Always schedule the next update of retry token bucket, even if the request count is 0 as the traffic + * can come back later, and we want to make sure that the retry token bucket is updated according to the + * new traffic volume. + */ + scheduler.schedule(this::updateRetryTokenBucket, enforcementWindowInMs, TimeUnit.MILLISECONDS); long elapsedTimeInMs = clock.millis() - lastUpdateTimestamp; - long requestCountSinceLastUpdate = requestCount.getAndSet(0); lastUpdateTimestamp = clock.millis(); - // Minimum user request per second will be 1 - long newQPS = (long) Math - .ceil((double) requestCountSinceLastUpdate / (double) TimeUnit.MILLISECONDS.toSeconds(elapsedTimeInMs)); - if (previousQPS > 0) { - long difference = Math.abs(previousQPS - newQPS); - double differenceInPercentDecimal = (double) difference / (double) previousQPS; - if (differenceInPercentDecimal > 0.1) { - // Only update the retry token bucket if the change in request per seconds is more than 10 percent + if (requestCount.get() > 0) { + long requestCountSinceLastUpdate = requestCount.getAndSet(0); + // Minimum user request per second will be 1 + long newQPS = (long) Math + .ceil((double) requestCountSinceLastUpdate / (double) TimeUnit.MILLISECONDS.toSeconds(elapsedTimeInMs)); + if (previousQPS > 0) { + long difference = Math.abs(previousQPS - newQPS); + double differenceInPercentDecimal = (double) difference / (double) previousQPS; + if (differenceInPercentDecimal > 0.1) { + // Only update the retry token bucket if the change in request per seconds is more than 10 percent + previousQPS = newQPS; + updateTokenBucket(newQPS); + } + } else { previousQPS = newQPS; updateTokenBucket(newQPS); } - } else { - previousQPS = newQPS; - updateTokenBucket(newQPS); } - scheduler.schedule(this::updateRetryTokenBucket, enforcementWindowInMs, TimeUnit.MILLISECONDS); } catch (Throwable e) { LOGGER.warn("Caught exception when trying to update retry budget, retry budget will be disabled", e); // Once disabled it will not be re-enabled until client is restarted diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/offsets/OffsetRecord.java b/internal/venice-common/src/main/java/com/linkedin/venice/offsets/OffsetRecord.java index 7e925e5a1ef..7d89138d921 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/offsets/OffsetRecord.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/offsets/OffsetRecord.java @@ -305,6 +305,15 @@ public void setPendingReportIncPushVersionList(List incPushVersionList) partitionState.pendingReportIncrementalPushVersions = new ArrayList<>(incPushVersionList); } + public Integer getRecordTransformerClassHash() { + Integer classHash = partitionState.getRecordTransformerClassHash(); + return classHash; + } + + public void setRecordTransformerClassHash(int classHash) { + this.partitionState.setRecordTransformerClassHash(classHash); + } + /** * It may be useful to cache this mapping. TODO: Explore GC tuning later. * @@ -321,7 +330,8 @@ public String toString() { + getPartitionUpstreamOffsetString() + ", leaderTopic=" + getLeaderTopic() + ", offsetLag=" + getOffsetLag() + ", eventTimeEpochMs=" + getMaxMessageTimeInMs() + ", latestProducerProcessingTimeInMs=" + getLatestProducerProcessingTimeInMs() + ", isEndOfPushReceived=" + isEndOfPushReceived() + ", databaseInfo=" - + getDatabaseInfo() + ", realTimeProducerState=" + getRealTimeProducerState() + '}'; + + getDatabaseInfo() + ", realTimeProducerState=" + getRealTimeProducerState() + ", recordTransformerClassHash=" + + getRecordTransformerClassHash() + '}'; } /** diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/adapter/kafka/consumer/ApacheKafkaConsumerAdapter.java b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/adapter/kafka/consumer/ApacheKafkaConsumerAdapter.java index 2d050ba6229..753716ece09 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/adapter/kafka/consumer/ApacheKafkaConsumerAdapter.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/adapter/kafka/consumer/ApacheKafkaConsumerAdapter.java @@ -5,11 +5,13 @@ import com.linkedin.venice.message.KafkaKey; import com.linkedin.venice.offsets.OffsetRecord; import com.linkedin.venice.pubsub.PubSubTopicPartitionInfo; +import com.linkedin.venice.pubsub.adapter.kafka.ApacheKafkaOffsetPosition; import com.linkedin.venice.pubsub.adapter.kafka.TopicPartitionsOffsetsTracker; import com.linkedin.venice.pubsub.api.PubSubConsumerAdapter; import com.linkedin.venice.pubsub.api.PubSubMessage; import com.linkedin.venice.pubsub.api.PubSubMessageDeserializer; import com.linkedin.venice.pubsub.api.PubSubMessageHeaders; +import com.linkedin.venice.pubsub.api.PubSubPosition; import com.linkedin.venice.pubsub.api.PubSubTopic; import com.linkedin.venice.pubsub.api.PubSubTopicPartition; import com.linkedin.venice.pubsub.api.exceptions.PubSubClientException; @@ -29,6 +31,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; @@ -88,8 +91,9 @@ public class ApacheKafkaConsumerAdapter implements PubSubConsumerAdapter { } /** - * Subscribe to a topic-partition if not already subscribed. If the topic-partition is already subscribed, this - * method is a no-op. This method requires the topic-partition to exist. + * Subscribes to a topic-partition if not already subscribed. If the topic-partition is already subscribed, + * this method is a no-op. + * * @param pubSubTopicPartition the topic-partition to subscribe to * @param lastReadOffset the last read offset for the topic-partition * @throws IllegalArgumentException if the topic-partition is null or the partition number is negative @@ -97,45 +101,90 @@ public class ApacheKafkaConsumerAdapter implements PubSubConsumerAdapter { */ @Override public void subscribe(PubSubTopicPartition pubSubTopicPartition, long lastReadOffset) { - String topic = pubSubTopicPartition.getPubSubTopic().getName(); - int partition = pubSubTopicPartition.getPartitionNumber(); - TopicPartition topicPartition = new TopicPartition(topic, partition); - Set topicPartitionSet = kafkaConsumer.assignment(); - if (topicPartitionSet.contains(topicPartition)) { - LOGGER.warn("Already subscribed to topic-partition:{}, ignoring subscription request", pubSubTopicPartition); - return; + subscribe( + pubSubTopicPartition, + (lastReadOffset <= OffsetRecord.LOWEST_OFFSET) + ? PubSubPosition.EARLIEST + : new ApacheKafkaOffsetPosition(lastReadOffset)); + } + + /** + * Subscribes to a specified topic-partition if it is not already subscribed. If the topic-partition is already + * subscribed, this method performs no action. + * + * The subscription uses the provided {@link PubSubPosition} to determine the starting offset for consumption. + * If the position is {@link PubSubPosition#EARLIEST}, the consumer will seek to the earliest available message. + * If it is {@link PubSubPosition#LATEST}, the consumer will seek to the latest offset. If an instance of + * {@link ApacheKafkaOffsetPosition} is provided, the consumer will seek to the specified offset plus one. + * + * @param pubSubTopicPartition the topic-partition to subscribe to + * @param lastReadPubSubPosition the last known position for the topic-partition + * @throws IllegalArgumentException if lastReadPubSubPosition is null or not an instance of {@link ApacheKafkaOffsetPosition} + * @throws PubSubTopicDoesNotExistException if the specified topic does not exist + */ + @Override + public void subscribe( + @Nonnull PubSubTopicPartition pubSubTopicPartition, + @Nonnull PubSubPosition lastReadPubSubPosition) { + if (lastReadPubSubPosition == null) { + LOGGER + .error("Failed to subscribe to topic-partition: {} because last read position is null", pubSubTopicPartition); + throw new IllegalArgumentException("Last read position cannot be null"); + } + if (lastReadPubSubPosition != PubSubPosition.EARLIEST && lastReadPubSubPosition != PubSubPosition.LATEST + && !(lastReadPubSubPosition instanceof ApacheKafkaOffsetPosition)) { + LOGGER.error( + "Failed to subscribe to topic-partition: {} because last read position type: {} is not supported with consumer type: {}", + pubSubTopicPartition, + lastReadPubSubPosition.getClass().getName(), + ApacheKafkaConsumerAdapter.class.getName()); + throw new IllegalArgumentException( + "Last read position must be an instance of " + ApacheKafkaOffsetPosition.class.getName() + " as consumer is " + + ApacheKafkaConsumerAdapter.class.getName() + " but it is " + + lastReadPubSubPosition.getClass().getName()); } - // Check if the topic-partition exists - if (config.shouldCheckTopicExistenceBeforeConsuming() && !isValidTopicPartition(pubSubTopicPartition)) { - LOGGER.error("Cannot subscribe to topic-partition: {} because it does not exist", pubSubTopicPartition); - throw new PubSubTopicDoesNotExistException(pubSubTopicPartition.getPubSubTopic()); + TopicPartition topicPartition = toKafkaTopicPartition(pubSubTopicPartition); + if (kafkaConsumer.assignment().contains(topicPartition)) { + LOGGER.warn( + "Already subscribed to topic-partition:{}, ignoring subscription request with position: {}", + pubSubTopicPartition, + lastReadPubSubPosition); + return; } - List topicPartitionList = new ArrayList<>(topicPartitionSet); + validateTopicExistence(pubSubTopicPartition); + + List topicPartitionList = new ArrayList<>(kafkaConsumer.assignment()); topicPartitionList.add(topicPartition); - kafkaConsumer.assign(topicPartitionList); // add the topic-partition to the subscription - // Use the last read offset to seek to the next offset to read. - long consumptionStartOffset = lastReadOffset <= OffsetRecord.LOWEST_OFFSET ? 0 : lastReadOffset + 1; - if (lastReadOffset <= OffsetRecord.LOWEST_OFFSET) { - if (lastReadOffset < OffsetRecord.LOWEST_OFFSET) { - LOGGER.warn( - "Last read offset: {} for topic-partition: {} is less than the lowest offset: {}, seeking to beginning." - + " This may indicate an off-by-one error.", - lastReadOffset, - pubSubTopicPartition, - OffsetRecord.LOWEST_OFFSET); - } + kafkaConsumer.assign(topicPartitionList); + + String logMessage; + if (lastReadPubSubPosition == PubSubPosition.EARLIEST) { kafkaConsumer.seekToBeginning(Collections.singletonList(topicPartition)); + logMessage = PubSubPosition.EARLIEST.toString(); + } else if (lastReadPubSubPosition == PubSubPosition.LATEST) { + kafkaConsumer.seekToEnd(Collections.singletonList(topicPartition)); + logMessage = PubSubPosition.LATEST.toString(); } else { + ApacheKafkaOffsetPosition kafkaOffsetPosition = (ApacheKafkaOffsetPosition) lastReadPubSubPosition; + long consumptionStartOffset = kafkaOffsetPosition.getOffset() + 1; kafkaConsumer.seek(topicPartition, consumptionStartOffset); + logMessage = "" + consumptionStartOffset; } assignments.put(topicPartition, pubSubTopicPartition); - LOGGER.info( - "Subscribed to topic-partition: {} at offset: {} and last read offset was: {}", - pubSubTopicPartition, - consumptionStartOffset, - lastReadOffset); + LOGGER.info("Subscribed to topic-partition: {} from position: {}", pubSubTopicPartition, logMessage); + } + + private TopicPartition toKafkaTopicPartition(PubSubTopicPartition topicPartition) { + return new TopicPartition(topicPartition.getPubSubTopic().getName(), topicPartition.getPartitionNumber()); + } + + private void validateTopicExistence(PubSubTopicPartition pubSubTopicPartition) { + if (config.shouldCheckTopicExistenceBeforeConsuming() && !isValidTopicPartition(pubSubTopicPartition)) { + LOGGER.error("Cannot subscribe to topic-partition: {} because it does not exist", pubSubTopicPartition); + throw new PubSubTopicDoesNotExistException(pubSubTopicPartition.getPubSubTopic()); + } } // visible for testing @@ -357,6 +406,13 @@ public long getOffsetLag(PubSubTopicPartition pubSubTopicPartition) { return topicPartitionsOffsetsTracker != null ? topicPartitionsOffsetsTracker.getOffsetLag(topic, partition) : -1; } + /** + * Returns the latest offset for the given topic-partition. The latest offsets are derived from the lag metric + * and may be outdated or imprecise. + * + * @param pubSubTopicPartition the topic-partition for which the latest offset is requested + * @return the latest offset, or -1 if tracking is unavailable + */ @Override public long getLatestOffset(PubSubTopicPartition pubSubTopicPartition) { String topic = pubSubTopicPartition.getPubSubTopic().getName(); diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/api/PubSubConsumerAdapter.java b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/api/PubSubConsumerAdapter.java index d6c1ba59085..5ea38e94da9 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/api/PubSubConsumerAdapter.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/api/PubSubConsumerAdapter.java @@ -1,5 +1,6 @@ package com.linkedin.venice.pubsub.api; +import com.linkedin.venice.annotation.UnderDevelopment; import com.linkedin.venice.kafka.protocol.KafkaMessageEnvelope; import com.linkedin.venice.message.KafkaKey; import com.linkedin.venice.pubsub.PubSubConstants; @@ -15,6 +16,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nonnull; /** @@ -38,6 +40,28 @@ public interface PubSubConsumerAdapter extends AutoCloseable, Closeable { */ void subscribe(PubSubTopicPartition pubSubTopicPartition, long lastReadOffset); + /** + * Subscribes to a specified topic-partition if it is not already subscribed. If the topic-partition is + * already subscribed, this method performs no action. + * + * The subscription uses the provided {@link PubSubPosition} to determine the starting offset for + * consumption. If the position is {@link PubSubPosition#EARLIEST}, the consumer will seek to the earliest + * available message. If it is {@link PubSubPosition#LATEST}, the consumer will seek to the latest available + * message. If a custom position is provided, implementations should resolve it to the corresponding offset + * or position in the underlying pub-sub system. + * + * Implementations of this interface should ensure proper validation of the topic-partition existence and + * manage consumer assignments. This method does not guarantee immediate subscription state changes and may + * defer them based on implementation details. + * + * @param pubSubTopicPartition the topic-partition to subscribe to + * @param lastReadPubSubPosition the last known position for the topic-partition + * @throws IllegalArgumentException if lastReadPubSubPosition is null or of an unsupported type + * @throws PubSubTopicDoesNotExistException if the specified topic does not exist + */ + @UnderDevelopment("This method is under development and may be subject to change.") + void subscribe(@Nonnull PubSubTopicPartition pubSubTopicPartition, @Nonnull PubSubPosition lastReadPubSubPosition); + /** * Unsubscribes the consumer from a specified topic-partition. * If the consumer was previously subscribed to the given partition, it will be unsubscribed, diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/api/PubSubPosition.java b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/api/PubSubPosition.java index 0cd1ce8d48c..3b248f7f230 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/api/PubSubPosition.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/pubsub/api/PubSubPosition.java @@ -7,6 +7,82 @@ * Represents a position of a message in a partition of a topic. */ public interface PubSubPosition { + /** + * A special position representing the earliest available message in a partition. All pub-sub adapters must support + * this position, and all pub-sub client implementations should interpret it as the earliest retrievable message in + * the partition. Implementations must map this position to the corresponding earliest offset or equivalent marker + * in the underlying pub-sub system. + */ + PubSubPosition EARLIEST = new PubSubPosition() { + @Override + public int comparePosition(PubSubPosition other) { + throw new IllegalStateException("Cannot compare EARLIEST position"); + } + + @Override + public long diff(PubSubPosition other) { + throw new IllegalStateException("Cannot diff EARLIEST position"); + } + + @Override + public PubSubPositionWireFormat getPositionWireFormat() { + throw new IllegalStateException("Cannot serialize EARLIEST position"); + } + + @Override + public String toString() { + return "EARLIEST"; + } + + @Override + public boolean equals(Object obj) { + return obj == this; + } + + @Override + public int hashCode() { + return -1; + } + }; + + /** + * A special position representing the latest available message in a partition. All pub-sub adapters must support + * this position, and all pub-sub client implementations should interpret it as the most recent retrievable message + * in the partition. Implementations must map this position to the corresponding latest offset or equivalent marker + * in the underlying pub-sub system. + */ + PubSubPosition LATEST = new PubSubPosition() { + @Override + public int comparePosition(PubSubPosition other) { + throw new IllegalStateException("Cannot compare LATEST position"); + } + + @Override + public long diff(PubSubPosition other) { + throw new IllegalStateException("Cannot diff LATEST position"); + } + + @Override + public PubSubPositionWireFormat getPositionWireFormat() { + throw new IllegalStateException("Cannot serialize LATEST position"); + } + + @Override + public String toString() { + return "LATEST"; + } + + @Override + public boolean equals(Object obj) { + return obj == this; + } + + @Override + public int hashCode() { + return -2; + } + }; + /** * @param other the other position to compare to * @return returns 0 if the positions are equal, diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/serialization/avro/AvroProtocolDefinition.java b/internal/venice-common/src/main/java/com/linkedin/venice/serialization/avro/AvroProtocolDefinition.java index e2e2bbc985b..44538ec0f47 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/serialization/avro/AvroProtocolDefinition.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/serialization/avro/AvroProtocolDefinition.java @@ -52,7 +52,7 @@ public enum AvroProtocolDefinition { * Used to persist the state of a partition in Storage Nodes, including offset, * Data Ingest Validation state, etc. */ - PARTITION_STATE(24, 14, PartitionState.class), + PARTITION_STATE(24, 15, PartitionState.class), /** * Used to persist state related to a store-version, including Start of Buffer Replay diff --git a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java index 11e10696296..29852764347 100644 --- a/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java +++ b/internal/venice-common/src/main/java/com/linkedin/venice/utils/Utils.java @@ -304,24 +304,49 @@ public static long parseLongFromString(String value, String fieldName) { } /** - * Since {@link Boolean#parseBoolean(String)} does not throw exception and will always return 'false' for - * any string that are not equal to 'true', We validate the string by our own. + * Parses a boolean from a string, ensuring that only valid boolean values ("true" or "false") + * are accepted. Throws an exception if the value is null or invalid. + * + * @param value the string to parse + * @param fieldName the name of the field being validated + * @return the parsed boolean value + * @throws VeniceHttpException if the value is null or not "true" or "false" */ - public static boolean parseBooleanFromString(String value, String fieldName) { + public static boolean parseBooleanOrThrow(String value, String fieldName) { if (value == null) { throw new VeniceHttpException( HttpStatus.SC_BAD_REQUEST, - fieldName + " must be a boolean, but value is null", + fieldName + " must be a boolean, but value is null.", ErrorType.BAD_REQUEST); } - if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) { - return Boolean.parseBoolean(value); - } else { + return parseBoolean(value, fieldName); + } + + /** + * Parses a boolean from a string, ensuring that only null and valid boolean values ("true" or "false") + * are accepted. Returns false if the value is null. + * + * @param value the string to parse + * @param fieldName the name of the field being validated + * @return the parsed boolean value, or false if the input is null + * @throws VeniceHttpException if the value is not "true" or "false" + */ + public static boolean parseBooleanOrFalse(String value, String fieldName) { + return value != null && parseBoolean(value, fieldName); + } + + /** + * Validates the boolean string, allowing only "true" or "false". + * Throws an exception if the value is invalid. + */ + private static boolean parseBoolean(String value, String fieldName) { + if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { throw new VeniceHttpException( HttpStatus.SC_BAD_REQUEST, - fieldName + " must be a boolean, but value: " + value, + fieldName + " must be a boolean, but value: " + value + " is invalid.", ErrorType.BAD_REQUEST); } + return Boolean.parseBoolean(value); } /** diff --git a/internal/venice-common/src/main/resources/avro/KafkaMessageEnvelope/v12/KafkaMessageEnvelope.avsc b/internal/venice-common/src/main/resources/avro/KafkaMessageEnvelope/v12/KafkaMessageEnvelope.avsc new file mode 100644 index 00000000000..29a12ec6892 --- /dev/null +++ b/internal/venice-common/src/main/resources/avro/KafkaMessageEnvelope/v12/KafkaMessageEnvelope.avsc @@ -0,0 +1,368 @@ +{ + "name": "KafkaMessageEnvelope", + "namespace": "com.linkedin.venice.kafka.protocol", + "type": "record", + "fields": [ + { + "name": "messageType", + "doc": "Using int because Avro Enums are not evolvable. Readers should always handle the 'unknown' value edge case, to account for future evolutions of this protocol. The mapping is the following: 0 => Put, 1 => Delete, 2 => ControlMessage, 3 => Update.", + "type": "int" + }, { + "name": "producerMetadata", + "doc": "ProducerMetadata contains information that the consumer can use to identify an upstream producer. This is common for all MessageType.", + "type": { + "name": "ProducerMetadata", + "type": "record", + "fields": [ + { + "name": "producerGUID", + "doc": "A unique identifier for this producer.", + "type": { + "name": "GUID", + "type": "fixed", + "size": 16 + } + }, { + "name": "segmentNumber", + "doc": "A number used to disambiguate between sequential segments sent into a given partition by a given producer. An incremented SegmentNumber should only be sent following an EndOfSegment control message. For finite streams (such as those bulk-loaded from Hadoop), it can be acceptable to have a single SegmentNumber per producer/partition combination, though that is not something that the downstream consumer should assume. For infinite streams, segments should be terminated and begun anew periodically. This number begins at 0.", + "type": "int" + }, { + "name": "messageSequenceNumber", + "doc": "A monotonically increasing number with no gaps used to distinguish unique messages produced in this segment (i.e.: by this producer into a given partition). This number begins at 0 (with a StartOfSegment ControlMessage) and subsequent messages (such as Put) will have a SequenceNumber of 1 and so forth.", + "type": "int" + }, { + "name": "messageTimestamp", + "doc": "The time of the producer's local system clock, at the time the message was submitted for production. This is the number of milliseconds from the unix epoch, 1 January 1970 00:00:00.000 UTC.", + "type": "long" + }, { + "name": "logicalTimestamp", + "doc": "This timestamp may be specified by the user. Sentinel value of -1 => apps are not using latest lib, -2 => apps have not specified the time. In case of negative values messageTimestamp field will be used for replication metadata.", + "type": "long", + "default": -1 + } + ] + } + }, { + "name": "payloadUnion", + "doc": "This contains the main payload of the message. Which branch of the union is present is based on the previously-defined MessageType field.", + "type": [ + { + "name": "Put", + "doc": "Put payloads contain a record value, and information on how to deserialize it.", + "type": "record", + "fields": [ + { + "name": "putValue", + "doc": "The record's value to be persisted in the storage engine.", + "type": "bytes" + }, { + "name": "schemaId", + "doc": "An identifier used to determine how the PutValue can be deserialized. Also used, in conjunction with the replicationMetadataVersionId, to deserialize the replicationMetadataPayload.", + "type": "int" + }, { + "name": "replicationMetadataVersionId", + "doc": "The A/A replication metadata schema version ID that will be used to deserialize replicationMetadataPayload.", + "type": "int", + "default": -1 + }, { + "name": "replicationMetadataPayload", + "doc": "The serialized value of the replication metadata schema.", + "type": "bytes", + "default": "" + } + ] + }, { + "name": "Update", + "doc": "Partial update operation, which merges the update value with the existing value.", + "type": "record", + "fields": [ + { + "name": "schemaId", + "doc": "The original schema ID.", + "type": "int" + }, { + "name": "updateSchemaId", + "doc": "The derived schema ID that will be used to deserialize updateValue.", + "type": "int" + }, { + "name": "updateValue", + "doc": "New value(s) for parts of the record that need to be updated.", + "type": "bytes" + } + ] + }, { + "name": "Delete", + "doc": "Delete payloads contain fields related to replication metadata of the record.", + "type": "record", + "fields": [ + { + "name": "schemaId", + "doc": "An identifier used, in conjunction with the replicationMetadataVersionId, to deserialize the replicationMetadataPayload.", + "type": "int", + "default": -1 + }, { + "name": "replicationMetadataVersionId", + "doc": "The A/A replication metadata schema version ID that will be used to deserialize replicationMetadataPayload.", + "type": "int", + "default": -1 + }, { + "name": "replicationMetadataPayload", + "doc": "The serialized value of the replication metadata schema.", + "type": "bytes", + "default": "" + } + ] + }, { + "name": "ControlMessage", + "doc": "ControlMessage payloads contain metadata about the stream of data, for validation and debuggability purposes.", + "type": "record", + "fields": [ + { + "name": "controlMessageType", + "doc": "Using int because Avro Enums are not evolvable. Readers should always handle the 'unknown' value edge case, to account for future evolutions of this protocol. The mapping is the following: 0 => StartOfPush, 1 => EndOfPush, 2 => StartOfSegment, 3 => EndOfSegment, 4 => StartOfBufferReplay (Deprecated), 5 => StartOfIncrementalPush, 6 => EndOfIncrementalPush, 7 => TopicSwitch, 8 => VersionSwap", + "type": "int" + }, { + "name": "debugInfo", + "doc": "This metadata is for logging and traceability purposes. It can be used to propagate information about the producer, the environment it runs in, or the source of data being produced into Venice. There should be no assumptions that any of this data will be used (or even looked at) by the downstream consumer in any particular way.", + "type": { + "type": "map", + "values": "string" + } + }, { + "name": "controlMessageUnion", + "doc": "This contains the ControlMessage data which is specific to each type of ControlMessage. Which branch of the union is present is based on the previously-defined MessageType field.", + "type": [ + { + "name": "StartOfPush", + "doc": "This ControlMessage is sent once per partition, at the beginning of a bulk load, before any of the data producers come online. This does not contain any data beyond the one which is common to all ControlMessageType.", + "type": "record", + "fields": [ + { + "name": "sorted", + "doc": "Whether the messages inside current topic partition between 'StartOfPush' control message and 'EndOfPush' control message is lexicographically sorted by key bytes", + "type": "boolean", + "default": false + }, { + "name": "chunked", + "doc": "Whether the messages inside the current push are encoded with chunking support. If true, this means keys will be prefixed with ChunkId, and values may contain a ChunkedValueManifest (if schema is defined as -20).", + "type": "boolean", + "default": false + }, { + "name": "compressionStrategy", + "doc": "What type of compression strategy the current push uses. Using int because Avro Enums are not evolvable. The mapping is the following: 0 => NO_OP, 1 => GZIP, 2 => ZSTD, 3 => ZSTD_WITH_DICT", + "type": "int", + "default": 0 + }, { + "name": "compressionDictionary", + "doc": "The raw bytes of dictionary used to compress/decompress records.", + "type": ["null", "bytes"], + "default": null + }, { + "name": "timestampPolicy", + "doc": "The policy to determine timestamps of batch push records. 0 => no per record replication metadata is stored, hybrid writes always win over batch, 1 => no per record timestamp metadata is stored, Start-Of-Push Control message's logicalTimestamp is treated as last update timestamp for all batch record, and hybrid writes wins only when their own logicalTimestamp are higher, 2 => per record timestamp metadata is provided by the push job and stored for each key, enabling full conflict resolution granularity on a per field basis, just like when merging concurrent update operations.", + "type": "int", + "default": 0 + } + ] + }, { + "name": "EndOfPush", + "doc": "This ControlMessage is sent once per partition, at the end of a bulk load, after all of the data producers come online. This does not contain any data beyond the one which is common to all ControlMessageType.", + "type": "record", + "fields": [] + }, { + "name": "StartOfSegment", + "doc": "This ControlMessage is sent at least once per partition per producer. It may be sent more than once per partition/producer, but only after the producer has sent an EndOfSegment into that partition to terminate the previously started segment.", + "type": "record", + "fields": [ + { + "name": "checksumType", + "doc": "Using int because Avro Enums are not evolvable. Readers should always handle the 'unknown' value edge case, to account for future evolutions of this protocol. The downstream consumer is expected to compute this checksum and use it to validate the incoming stream of data. The current mapping is the following: 0 => None, 1 => MD5, 2 => Adler32, 3 => CRC32.", + "type": "int" + }, { + "name": "upcomingAggregates", + "doc": "An array of names of aggregate computation strategies for which there will be a value percolated in the corresponding EndOfSegment ControlMessage. The downstream consumer may choose to compute these aggregates on its own and use them as additional validation safeguards, or it may choose to merely log them, or even ignore them altogether.", + "type": { + "type": "array", + "items": "string" + } + } + ] + }, { + "name": "EndOfSegment", + "doc": "This ControlMessage is sent at least once per partition per producer. It may be sent more than once per partition/producer, but only after the producer has sent a StartOfSegment into that partition. There should be an equal number of StartOfSegment and EndOfSegment messages in each producer/partition pair.", + "type": "record", + "fields": [ + { + "name": "checksumValue", + "doc": "The value of the checksum computed since the last StartOfSegment ControlMessage.", + "type": "bytes" + }, { + "name": "computedAggregates", + "doc": "A map containing the results of the aggregate computation strategies that were promised in the previous StartOfSegment ControlMessage. The downstream consumer may choose to compare the value of these aggregates against those that it computed on its own ir oder to use them as additional validation safeguards, or it may choose to merely log them, or even ignore them altogether.", + "type": { + "type": "array", + "items": "long" + } + }, { + "name": "finalSegment", + "doc": "This field is set to true when the producer knows that there is no more data coming from its data source after this EndOfSegment. This happens at the time the producer is closed.", + "type": "boolean" + } + ] + }, { + "name": "StartOfBufferReplay", + "doc": "[Deprecated] This ControlMessage is sent by the Controller, once per partition, after the EndOfPush ControlMessage, in Hybrid Stores that ingest from both offline and nearline sources. It contains information about the the offsets from which the Buffer Replay Service started replaying data from the real-time buffer topic onto the store-version topic. This can be used as a synchronization marker between the real-time buffer topic and the store-version topic, akin to how a clapperboard is used to synchronize sound and image in filmmaking. This synchronization marker can in turn be used by the consumer to compute an offset lag.", + "type": "record", + "fields": [ + { + "name": "sourceOffsets", + "doc": "Array of offsets from the real-time buffer topic at which the Buffer Replay Service started replaying data. The index position of the array corresponds to the partition number in the real-time buffer.", + "type": { + "type": "array", + "items": "long" + } + }, { + "name": "sourceKafkaCluster", + "doc": "Kafka bootstrap servers URL of the cluster where the source buffer exists.", + "type": "string" + }, { + "name": "sourceTopicName", + "doc": "Name of the source buffer topic.", + "type": "string" + } + ] + }, { + "name": "StartOfIncrementalPush", + "doc": "This ControlMessage is sent per partition by each offline incremental push job, once per partition, at the beginning of a incremental push.", + "type": "record", + "fields": [ + { + "name": "version", + "doc": "The version of current incremental push. Each incremental push is associated with a version. Both 'StartOfIncrementalPush' control message and 'EndOfIncrementalPush' contain version info so they can be paired to each other.", + "type": "string" + } + ] + }, { + "name": "EndOfIncrementalPush", + "doc": "This ControlMessage is sent per partition by each offline incremental push job, once per partition, at the end of a incremental push", + "type": "record", + "fields": [ + { + "name": "version", + "doc": "The version of current incremental push. Each incremental push is associated with a version. Both 'StartOfIncrementalPush' control message and 'EndOfIncrementalPush' contain version info so they can be paired to each other.", + "type": "string" + } + ] + }, { + "name": "TopicSwitch", + "doc": "This ControlMessage is sent by the Controller, once per partition; it will only be used in leader/follower state transition model; this control message will indicate the leader to switch to a new source topic and start consuming from offset with a specific timestamp.", + "type": "record", + "fields": [ + { + "name": "sourceKafkaServers", + "doc": "A list of Kafka bootstrap servers URLs where the new source topic exists; currently there will be only one URL in the list, but the list opens up the possibility for leader to consume from different fabrics in active-active replication mode.", + "type": { + "type": "array", + "items": "string" + } + }, { + "name": "sourceTopicName", + "doc": "Name of new the source topic.", + "type": "string" + }, { + "name": "rewindStartTimestamp", + "doc": "The creation time of this control message in parent controller minus the rewind time of the corresponding store; leaders in different fabrics will get the offset of the source topic by the same start timestamp and start consuming from there; if timestamp is 0, leader will start consuming from the beginning of the source topic.", + "type": "long" + } + ] + }, { + "name": "VersionSwap", + "doc": "This controlMessage is written to the real-time topic by the controller or to the store-version topic by the current version's leader server. It can be used to let current version and future version synchronize on a specific point for all regions' real-time topics, to guarantee there is only one store version producing to change capture topic all the time. It can also be used by the consumer client to switch to another store-version topic and filter messages that have a lower watermark than the one dictated by the leader.", + "type": "record", + "fields": [ + { + "name": "oldServingVersionTopic", + "doc": "Name of the old source topic we are switching from.", + "type": "string" + }, { + "name": "newServingVersionTopic", + "doc": "Name of the new source topic we are switching to.", + "type": "string" + }, { + "name": "localHighWatermarks", + "doc": "The latest offsets of all real-time topic has been consumed up until now.", + "type": [ + "null", + { + "type": "array", + "items": "long" + } + ], + "default": null + }, { + "name": "localHighWatermarkPubSubPositions", + "doc": "The latest pubsub positions of all real-time topics consumed up until now.", + "type": { + "type": "array", + "items": "bytes" + }, + "default": [] + }, { + "name": "isRepush", + "doc": "Flag to indicate this version swap is triggered by repush or not.", + "type": "boolean", + "default": false + }, { + "name": "isLastVersionSwapMessageFromRealTimeTopic", + "doc": "Flag to indicate this version swap message in version topic is triggered by the last version swap in real time topic the leader server has received. With this flag, new leader will be able to recover the full state during leadership handover, when we rely on real-time topics for all regions to achieve version swap synchronization.", + "type": "boolean", + "default": false + } + ] + } + ] + } + ] + } + ] + }, { + "name": "leaderMetadataFooter", + "doc": "A optional footer that leader SN can use to give extra L/F related mete data", + "type": [ + "null", + { + "name": "LeaderMetadata", + "type": "record", + "fields": [ + { + "name": "hostName", + "doc": "The identifier of the host which sends the message.This helps detect the 'split brain' scenario in leader SN. Notice that it is different from GUID. GUID represents the one who produces the message. In 'pass-through' mode, the relaying producer will reuse the same GUID from the upstream message.", + "type": "string" + }, { + "name": "upstreamOffset", + "doc": "Where this message is located in RT/GF/remote VT topic. This value will be determined and modified by leader SN at runtime.", + "type": "long", + "default": -1 + }, { + "name": "upstreamKafkaClusterId", + "doc": "Kafka bootstrap server URL of the cluster where RT/GF/remote VT topic exists, represented by an integer to reduce the overhead. This value will be determined and modified by leader SN at runtime.", + "type": "int", + "default": -1 + }, { + "name": "upstreamPubSubPosition", + "doc": "The position of the message in the upstream pubsub system (usually real-time topic).", + "type": "bytes", + "default": "" + }, { + "name": "termId", + "doc": "TermId is a unique identifier (usually a Helix message timestamp) for the term in which the message is produced.", + "type": "long", + "default": -1 + } + ] + } + ], + "default": null + } + ] +} \ No newline at end of file diff --git a/internal/venice-common/src/main/resources/avro/PartitionState/v15/PartitionState.avsc b/internal/venice-common/src/main/resources/avro/PartitionState/v15/PartitionState.avsc new file mode 100644 index 00000000000..854c5f42116 --- /dev/null +++ b/internal/venice-common/src/main/resources/avro/PartitionState/v15/PartitionState.avsc @@ -0,0 +1,200 @@ +{ + "name": "PartitionState", + "namespace": "com.linkedin.venice.kafka.protocol.state", + "doc": "This record holds the state necessary for a consumer to checkpoint its progress when consuming a Venice partition. When provided the state in this record, a consumer should thus be able to resume consuming midway through a stream.", + "type": "record", + "fields": [ + { + "name": "offset", + "doc": "The last Kafka offset consumed successfully in this partition from version topic.", + "type": "long" + }, { + "name": "offsetLag", + "doc": "The last Kafka offset lag in this partition for fast online transition in server restart.", + "type": "long", + "default": -1 + }, { + "name": "endOfPush", + "doc": "Whether the EndOfPush control message was consumed in this partition.", + "type": "boolean" + }, { + "name": "lastUpdate", + "doc": "The last time this PartitionState was updated. Can be compared against the various messageTimestamp in ProducerPartitionState in order to infer lag time between producers and the consumer maintaining this PartitionState.", + "type": "long" + }, { + "name": "startOfBufferReplayDestinationOffset", + "doc": "This is the offset at which the StartOfBufferReplay control message was consumed in the current partition of the destination topic. This is not the same value as the source offsets contained in the StartOfBufferReplay control message itself. The source and destination offsets act together as a synchronization marker. N.B.: null means that the SOBR control message was not received yet.", + "type": ["null", "long"], + "default": null + }, { + "name": "databaseInfo", + "doc": "A map of string -> string to store database related info, which is necessary to checkpoint", + "type": { + "type": "map", + "values": "string" + }, + "default": {} + }, { + "name": "incrementalPushInfo", + "doc": "metadata of ongoing incremental push in the partition", + "type": [ + "null", + { + "name": "IncrementalPush", + "type": "record", + "fields": [ + { + "name": "version", + "doc": "The version of current incremental push", + "type": "string" + }, { + "name": "error", + "doc": "The first error founded during in`gestion", + "type": ["null", "string"], + "default": null + } + ] + } + ], + "default": null + }, { + "name": "leaderTopic", + "type": ["null", "string"], + "doc": "The topic that leader is consuming from; for leader, leaderTopic can be different from the version topic; for follower, leaderTopic is the same as version topic.", + "default": null + }, { + "name": "leaderOffset", + "type": "long", + "doc": "The last Kafka offset consumed successfully in this partition from the leader topic. TODO: remove this field once upstreamOffsetMap is used everywhere.", + "default": -1 + }, { + "name": "upstreamOffsetMap", + "type": { + "type": "map", + "values": "long", + "java-key-class": "java.lang.String", + "avro.java.string": "String" + }, + "doc": "A map of upstream Kafka bootstrap server url -> the last Kafka offset consumed from upstream topic.", + "default": {} + }, { + "name": "upstreamVersionTopicOffset", + "type": "long", + "doc": "The last upstream version topic offset persisted to disk; if the batch native-replication source is the same as local region, this value will always be -1", + "default": -1 + }, { + "name": "leaderGUID", + "type": [ + "null", + { + "namespace": "com.linkedin.venice.kafka.protocol", + "name": "GUID", + "type": "fixed", + "size": 16 + } + ], + "doc": "This field is deprecated since GUID is no longer able to identify the split-brain issue once 'pass-through' mode is enabled in venice writer. The field is superseded by leaderHostId and will be removed in the future ", + "default": null + }, { + "name": "leaderHostId", + "type": ["null", "string"], + "doc": "An unique identifier (such as host name) that stands for each host. It's used to identify if there is a split-brain happened while the leader(s) re-produce records", + "default": null + }, { + "name": "producerStates", + "doc": "A map of producer GUID -> producer state.", + "type": { + "type": "map", + "values": { + "name": "ProducerPartitionState", + "doc": "A record containing the state pertaining to the data sent by one upstream producer into one partition.", + "type": "record", + "fields": [ + { + "name": "segmentNumber", + "doc": "The current segment number corresponds to the last (highest) segment number for which we have seen a StartOfSegment control message.", + "type": "int" + }, { + "name": "segmentStatus", + "doc": "The status of the current segment: 0 => NOT_STARTED, 1 => IN_PROGRESS, 2 => END_OF_INTERMEDIATE_SEGMENT, 3 => END_OF_FINAL_SEGMENT.", + "type": "int" + }, { + "name": "isRegistered", + "doc": "Whether the segment is registered. i.e. received Start_Of_Segment to initialize the segment.", + "type": "boolean", + "default": false + }, { + "name": "messageSequenceNumber", + "doc": "The current message sequence number, within the current segment, which we have seen for this partition/producer pair.", + "type": "int" + }, { + "name": "messageTimestamp", + "doc": "The timestamp included in the last message we have seen for this partition/producer pair.", + "type": "long" + }, { + "name": "checksumType", + "doc": "The current mapping is the following: 0 => None, 1 => MD5, 2 => Adler32, 3 => CRC32.", + "type": "int" + }, { + "name": "checksumState", + "doc": "The value of the checksum computed since the last StartOfSegment ControlMessage.", + "type": "bytes" + }, { + "name": "aggregates", + "doc": "The aggregates that have been computed so far since the last StartOfSegment ControlMessage.", + "type": { + "type": "map", + "values": "long" + } + }, { + "name": "debugInfo", + "doc": "The debug info received as part of the last StartOfSegment ControlMessage.", + "type": { + "type": "map", + "values": "string" + } + } + ] + } + } + }, { + "name": "previousStatuses", + "doc": "A map of string -> string which stands for previous PartitionStatus", + "type": { + "type": "map", + "values": "string" + }, + "default": {} + }, { + "name": "pendingReportIncrementalPushVersions", + "doc": "A list of string which stands for incremental push versions which have received EOIP but not yet reported prior to lag caught up, they will be reported in batch", + "type": { + "type": "array", + "items": "string" + }, + "default": [] + }, { + "name": "realtimeTopicProducerStates", + "doc": "A map that maps upstream Kafka bootstrap server url -> to a map of producer GUID -> producer state for real-time data.", + "type": { + "type": "map", + "values": { + "type": "map", + "values": "com.linkedin.venice.kafka.protocol.state.ProducerPartitionState" + }, + "java-key-class": "java.lang.String", + "avro.java.string": "String" + }, + "default": {} + }, + { + "name": "recordTransformerClassHash", + "doc": "An integer hash code used by the DaVinciRecordTransformer to detect changes to the user's class during bootstrapping.", + "type": [ + "null", + "int" + ], + "default": null + } + ] +} diff --git a/internal/venice-common/src/main/resources/avro/PartitionState/v16/PartitionState.avsc b/internal/venice-common/src/main/resources/avro/PartitionState/v16/PartitionState.avsc new file mode 100644 index 00000000000..d1c3ef6eeb6 --- /dev/null +++ b/internal/venice-common/src/main/resources/avro/PartitionState/v16/PartitionState.avsc @@ -0,0 +1,230 @@ +{ + "name": "PartitionState", + "namespace": "com.linkedin.venice.kafka.protocol.state", + "doc": "This record holds the state necessary for a consumer to checkpoint its progress when consuming a Venice partition. When provided the state in this record, a consumer should thus be able to resume consuming midway through a stream.", + "type": "record", + "fields": [ + { + "name": "offset", + "doc": "The last Kafka offset consumed successfully in this partition from version topic.", + "type": "long" + }, { + "name": "lastProcessedVersionTopicPubSubPosition", + "doc": "The last PubSub position consumed successfully in this partition from version topic", + "type": "bytes", + "default": "" + }, { + "name": "latestObservedTermId", + "doc": "The most recent termId observed by this replica.", + "type": "long", + "default": -1 + }, { + "name": "currentTermStartPubSubPosition", + "doc": "The pubsub position marking the start of the current leader's term in the version topic.", + "type": "bytes", + "default": "" + }, { + "name": "offsetLag", + "doc": "The last Kafka offset lag in this partition for fast online transition in server restart.", + "type": "long", + "default": -1 + }, { + "name": "endOfPush", + "doc": "Whether the EndOfPush control message was consumed in this partition.", + "type": "boolean" + }, { + "name": "lastUpdate", + "doc": "The last time this PartitionState was updated. Can be compared against the various messageTimestamp in ProducerPartitionState in order to infer lag time between producers and the consumer maintaining this PartitionState.", + "type": "long" + }, { + "name": "startOfBufferReplayDestinationOffset", + "doc": "This is the offset at which the StartOfBufferReplay control message was consumed in the current partition of the destination topic. This is not the same value as the source offsets contained in the StartOfBufferReplay control message itself. The source and destination offsets act together as a synchronization marker. N.B.: null means that the SOBR control message was not received yet.", + "type": ["null", "long"], + "default": null + }, { + "name": "databaseInfo", + "doc": "A map of string -> string to store database related info, which is necessary to checkpoint", + "type": { + "type": "map", + "values": "string" + }, + "default": {} + }, { + "name": "incrementalPushInfo", + "doc": "metadata of ongoing incremental push in the partition", + "type": [ + "null", + { + "name": "IncrementalPush", + "type": "record", + "fields": [ + { + "name": "version", + "doc": "The version of current incremental push", + "type": "string" + }, { + "name": "error", + "doc": "The first error founded during in`gestion", + "type": ["null", "string"], + "default": null + } + ] + } + ], + "default": null + }, { + "name": "leaderTopic", + "type": ["null", "string"], + "doc": "The topic that leader is consuming from; for leader, leaderTopic can be different from the version topic; for follower, leaderTopic is the same as version topic.", + "default": null + }, { + "name": "leaderOffset", + "type": "long", + "doc": "The last Kafka offset consumed successfully in this partition from the leader topic. TODO: remove this field once upstreamOffsetMap is used everywhere.", + "default": -1 + }, { + "name": "upstreamOffsetMap", + "type": { + "type": "map", + "values": "long", + "java-key-class": "java.lang.String", + "avro.java.string": "String" + }, + "doc": "A map of upstream Kafka bootstrap server url -> the last Kafka offset consumed from upstream topic.", + "default": {} + }, { + "name": "upstreamRealTimeTopicPubSubPositionMap", + "type": { + "type": "map", + "values": "bytes", + "java-key-class": "java.lang.String", + "avro.java.string": "String" + }, + "doc": "A map of upstream PubSub bootstrap server url -> the last PubSub position consumed from upstream topic.", + "default": {} + }, { + "name": "upstreamVersionTopicOffset", + "type": "long", + "doc": "The last upstream version topic offset persisted to disk; if the batch native-replication source is the same as local region, this value will always be -1", + "default": -1 + }, { + "name": "upstreamVersionTopicPubSubPosition", + "type": "bytes", + "doc": "The last upstream version topic PubSub position persisted to disk; if the batch native-replication source is the same as local region, this value will always be null", + "default": "" + }, { + "name": "leaderGUID", + "type": [ + "null", + { + "namespace": "com.linkedin.venice.kafka.protocol", + "name": "GUID", + "type": "fixed", + "size": 16 + } + ], + "doc": "This field is deprecated since GUID is no longer able to identify the split-brain issue once 'pass-through' mode is enabled in venice writer. The field is superseded by leaderHostId and will be removed in the future ", + "default": null + }, { + "name": "leaderHostId", + "type": ["null", "string"], + "doc": "An unique identifier (such as host name) that stands for each host. It's used to identify if there is a split-brain happened while the leader(s) re-produce records", + "default": null + }, { + "name": "producerStates", + "doc": "A map of producer GUID -> producer state.", + "type": { + "type": "map", + "values": { + "name": "ProducerPartitionState", + "doc": "A record containing the state pertaining to the data sent by one upstream producer into one partition.", + "type": "record", + "fields": [ + { + "name": "segmentNumber", + "doc": "The current segment number corresponds to the last (highest) segment number for which we have seen a StartOfSegment control message.", + "type": "int" + }, { + "name": "segmentStatus", + "doc": "The status of the current segment: 0 => NOT_STARTED, 1 => IN_PROGRESS, 2 => END_OF_INTERMEDIATE_SEGMENT, 3 => END_OF_FINAL_SEGMENT.", + "type": "int" + }, { + "name": "isRegistered", + "doc": "Whether the segment is registered. i.e. received Start_Of_Segment to initialize the segment.", + "type": "boolean", + "default": false + }, { + "name": "messageSequenceNumber", + "doc": "The current message sequence number, within the current segment, which we have seen for this partition/producer pair.", + "type": "int" + }, { + "name": "messageTimestamp", + "doc": "The timestamp included in the last message we have seen for this partition/producer pair.", + "type": "long" + }, { + "name": "checksumType", + "doc": "The current mapping is the following: 0 => None, 1 => MD5, 2 => Adler32, 3 => CRC32.", + "type": "int" + }, { + "name": "checksumState", + "doc": "The value of the checksum computed since the last StartOfSegment ControlMessage.", + "type": "bytes" + }, { + "name": "aggregates", + "doc": "The aggregates that have been computed so far since the last StartOfSegment ControlMessage.", + "type": { + "type": "map", + "values": "long" + } + }, { + "name": "debugInfo", + "doc": "The debug info received as part of the last StartOfSegment ControlMessage.", + "type": { + "type": "map", + "values": "string" + } + } + ] + } + } + }, { + "name": "previousStatuses", + "doc": "A map of string -> string which stands for previous PartitionStatus", + "type": { + "type": "map", + "values": "string" + }, + "default": {} + }, { + "name": "pendingReportIncrementalPushVersions", + "doc": "A list of string which stands for incremental push versions which have received EOIP but not yet reported prior to lag caught up, they will be reported in batch", + "type": { + "type": "array", + "items": "string" + }, + "default": [] + }, { + "name": "realtimeTopicProducerStates", + "doc": "A map that maps upstream Kafka bootstrap server url -> to a map of producer GUID -> producer state for real-time data.", + "type": { + "type": "map", + "values": { + "type": "map", + "values": "com.linkedin.venice.kafka.protocol.state.ProducerPartitionState" + }, + "java-key-class": "java.lang.String", + "avro.java.string": "String" + }, + "default": {} + }, + { + "name": "recordTransformerClassHash", + "doc": "An integer hash code used by the DaVinciRecordTransformer to detect changes to the user's class during bootstrapping.", + "type": [ + "null", + "int" + ], + "default": null + } + ] +} diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/blobtransfer/DaVinciBlobFinderTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/blobtransfer/DaVinciBlobFinderTest.java index 77bd60e8482..f8b136a0b86 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/blobtransfer/DaVinciBlobFinderTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/blobtransfer/DaVinciBlobFinderTest.java @@ -11,15 +11,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.venice.client.store.AbstractAvroStoreClient; +import com.linkedin.venice.client.store.ClientConfig; +import com.linkedin.venice.client.store.ClientFactory; +import com.linkedin.venice.client.store.transport.TransportClient; import com.linkedin.venice.client.store.transport.TransportClientResponse; import com.linkedin.venice.utils.ObjectMapperFactory; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.function.Function; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -35,12 +40,18 @@ public class DaVinciBlobFinderTest { @BeforeMethod public void setUp() { + ClientConfig clientConfig = ClientConfig.defaultGenericClientConfig(storeName); + storeClient = mock(AbstractAvroStoreClient.class); - daVinciBlobFinder = new DaVinciBlobFinder(storeClient); + daVinciBlobFinder = spy(new DaVinciBlobFinder(clientConfig)); + + Mockito.doReturn(storeName).when(storeClient).getStoreName(); } @Test public void testDiscoverBlobPeers_Success() { + Mockito.doReturn(storeClient).when(daVinciBlobFinder).getStoreClient(storeName); + String responseBodyJson = "{\"error\":false,\"errorMessage\":\"\",\"discoveryResult\":[\"host1\",\"host2\",\"host3\"]}"; byte[] responseBody = responseBodyJson.getBytes(StandardCharsets.UTF_8); @@ -57,6 +68,7 @@ public void testDiscoverBlobPeers_Success() { @Test public void testDiscoverBlobPeers_CallsTransportClientWithCorrectURI() { + Mockito.doReturn(storeClient).when(daVinciBlobFinder).getStoreClient(storeName); String responseBodyJson = "{\"error\":false,\"errorMessage\":\"\",\"discoveryResult\":[\"host1\",\"host2\",\"host3\"]}"; byte[] responseBody = responseBodyJson.getBytes(StandardCharsets.UTF_8); @@ -78,6 +90,8 @@ public void testDiscoverBlobPeers_CallsTransportClientWithCorrectURI() { @Test public void testDiscoverBlobPeers_ContentDeserializationError() throws Exception { + Mockito.doReturn(storeClient).when(daVinciBlobFinder).getStoreClient(storeName); + String responseBodyJson = "{\"error\":true,\"errorMessage\":\"some error\",\"discoveryResult\":[]}"; byte[] responseBody = responseBodyJson.getBytes(StandardCharsets.UTF_8); TransportClientResponse mockResponse = new TransportClientResponse(0, null, responseBody); @@ -98,6 +112,8 @@ public void testDiscoverBlobPeers_ContentDeserializationError() throws Exception @Test public void testDiscoverBlobPeers_ClientWithIncorrectUri() { + Mockito.doReturn(storeClient).when(daVinciBlobFinder).getStoreClient(storeName); + CompletableFuture futureResponse = new CompletableFuture<>(); futureResponse.completeExceptionally(new RuntimeException("Test Exception")); when(storeClient.getRaw(anyString())).thenReturn(futureResponse); @@ -109,4 +125,39 @@ public void testDiscoverBlobPeers_ClientWithIncorrectUri() { response.getErrorMessage(), "Error finding DVC peers for blob transfer in store: testStore, version: 1, partition: 1"); } + + @Test + public void testGetStoreClient() throws IOException { + // set up the transport client provider used to initialize the store client + TransportClient transportClient1 = mock(TransportClient.class); + TransportClient transportClient2 = mock(TransportClient.class); + Function clientConfigTransportClientFunction = (clientConfig) -> { + if (clientConfig.getStoreName().equals(storeName)) { + return transportClient1; + } else if (clientConfig.getStoreName().equals("storeName2")) { + return transportClient2; + } else { + // Create TransportClient the regular way + return null; + } + }; + ClientFactory.setUnitTestMode(); + ClientFactory.setTransportClientProvider(clientConfigTransportClientFunction); + + // ClientConfig is initialized with storeName + AbstractAvroStoreClient storeClient = daVinciBlobFinder.getStoreClient(storeName); + Assert.assertNotNull(storeClient); + Assert.assertEquals(storeClient.getStoreName(), storeName); + + // Even if the daVinciBlobFinder is initialized at the beginning with "storeName", the getStoreClient + // method should be able to return a store client for "storeName2" + AbstractAvroStoreClient storeClient2 = daVinciBlobFinder.getStoreClient("storeName2"); + Assert.assertNotNull(storeClient2); + Assert.assertEquals(storeClient2.getStoreName(), "storeName2"); + + daVinciBlobFinder.close(); + // verify that the store client is closed + verify(transportClient1).close(); + verify(transportClient2).close(); + } } diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/meta/RetryManagerTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/meta/RetryManagerTest.java index ae615b6abc5..bcf299ee264 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/meta/RetryManagerTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/meta/RetryManagerTest.java @@ -1,7 +1,9 @@ package com.linkedin.venice.meta; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import com.linkedin.venice.utils.TestUtils; import io.tehuti.metrics.MetricsRepository; @@ -69,6 +71,20 @@ public void testRetryManager() { // We should eventually be able to perform retries again TestUtils .waitForNonDeterministicAssertion(5, TimeUnit.SECONDS, () -> Assert.assertTrue(retryManager.isRetryAllowed())); + + // Make sure the empty request count kicks in + doReturn(start + 3001).when(mockClock).millis(); + TestUtils.waitForNonDeterministicAssertion(5, TimeUnit.SECONDS, () -> { + verify(mockClock, atLeast(10)).millis(); + }); + + // Send more traffic + doReturn(start + 4001).when(mockClock).millis(); + retryManager.recordRequests(100); + TestUtils.waitForNonDeterministicAssertion( + 5, + TimeUnit.SECONDS, + () -> Assert.assertTrue(retryManager.isRetryAllowed(40))); } @Test(timeOut = TEST_TIMEOUT_IN_MS) diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/adapter/kafka/consumer/ApacheKafkaConsumerAdapterTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/adapter/kafka/consumer/ApacheKafkaConsumerAdapterTest.java index 36f9f4d1612..ce74c5b0641 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/adapter/kafka/consumer/ApacheKafkaConsumerAdapterTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/pubsub/adapter/kafka/consumer/ApacheKafkaConsumerAdapterTest.java @@ -1,11 +1,14 @@ package com.linkedin.venice.pubsub.adapter.kafka.consumer; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -27,9 +30,11 @@ import com.linkedin.venice.pubsub.PubSubTopicPartitionImpl; import com.linkedin.venice.pubsub.PubSubTopicPartitionInfo; import com.linkedin.venice.pubsub.PubSubTopicRepository; +import com.linkedin.venice.pubsub.adapter.kafka.ApacheKafkaOffsetPosition; import com.linkedin.venice.pubsub.adapter.kafka.TopicPartitionsOffsetsTracker; import com.linkedin.venice.pubsub.api.PubSubMessage; import com.linkedin.venice.pubsub.api.PubSubMessageDeserializer; +import com.linkedin.venice.pubsub.api.PubSubPosition; import com.linkedin.venice.pubsub.api.PubSubTopic; import com.linkedin.venice.pubsub.api.PubSubTopicPartition; import com.linkedin.venice.pubsub.api.exceptions.PubSubClientException; @@ -96,6 +101,109 @@ public void cleanUp() { pubSubMessageDeserializer.close(); } + @Test + public void testSubscribeWithValidOffset() { + PubSubTopicPartition pubSubTopicPartition = new PubSubTopicPartitionImpl(pubSubTopicRepository.getTopic("test"), 0); + TopicPartition topicPartition = new TopicPartition("test", 0); + + when(internalKafkaConsumer.assignment()).thenReturn(Collections.emptySet()); + doNothing().when(internalKafkaConsumer).assign(any()); + + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, 100); + assertTrue(kafkaConsumerAdapter.getAssignment().contains(pubSubTopicPartition)); + verify(internalKafkaConsumer).assign(any(List.class)); + verify(internalKafkaConsumer).seek(topicPartition, 101); // Should seek to offset + 1 + + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, 200); + verify(internalKafkaConsumer, times(2)).assign(any(List.class)); + verify(internalKafkaConsumer).seek(topicPartition, 201); // Should seek to offset + 1 + } + + @Test + public void testSubscribeWithEarliestOffset() { + PubSubTopicPartition pubSubTopicPartition = new PubSubTopicPartitionImpl(pubSubTopicRepository.getTopic("test"), 0); + TopicPartition topicPartition = new TopicPartition("test", 0); + + when(internalKafkaConsumer.assignment()).thenReturn(Collections.emptySet()); + doNothing().when(internalKafkaConsumer).assign(any()); + + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, PubSubPosition.EARLIEST); + assertTrue(kafkaConsumerAdapter.getAssignment().contains(pubSubTopicPartition)); + verify(internalKafkaConsumer).assign(any(List.class)); + verify(internalKafkaConsumer).seekToBeginning(Collections.singletonList(topicPartition)); + + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, -1); + verify(internalKafkaConsumer, times(2)).assign(any(List.class)); + verify(internalKafkaConsumer, times(2)).seekToBeginning(Collections.singletonList(topicPartition)); + } + + @Test + public void testSubscribeAlreadySubscribed() { + PubSubTopicPartition pubSubTopicPartition = new PubSubTopicPartitionImpl(pubSubTopicRepository.getTopic("test"), 0); + TopicPartition topicPartition = new TopicPartition("test", 0); + + when(internalKafkaConsumer.assignment()).thenReturn(Collections.singleton(topicPartition)); + + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, 100); + + verify(internalKafkaConsumer, never()).assign(any()); + verify(internalKafkaConsumer, never()).seek(any(), anyLong()); + } + + @Test + public void testSubscribeWithLatestPubSubPosition() { + PubSubTopicPartition pubSubTopicPartition = new PubSubTopicPartitionImpl(pubSubTopicRepository.getTopic("test"), 0); + TopicPartition topicPartition = new TopicPartition("test", 0); + + when(internalKafkaConsumer.assignment()).thenReturn(Collections.emptySet()); + + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, PubSubPosition.LATEST); + + assertTrue(kafkaConsumerAdapter.getAssignment().contains(pubSubTopicPartition)); + verify(internalKafkaConsumer).assign(any(List.class)); + verify(internalKafkaConsumer).seekToEnd(Collections.singletonList(topicPartition)); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testSubscribeWithNullPubSubPosition() { + PubSubTopicPartition pubSubTopicPartition = new PubSubTopicPartitionImpl(pubSubTopicRepository.getTopic("test"), 0); + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testSubscribeWithInvalidPubSubPositionType() { + PubSubTopicPartition pubSubTopicPartition = new PubSubTopicPartitionImpl(pubSubTopicRepository.getTopic("test"), 0); + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, mock(PubSubPosition.class)); + } + + @Test + public void testSubscribeWithApacheKafkaOffsetPosition() { + PubSubTopicPartition pubSubTopicPartition = new PubSubTopicPartitionImpl(pubSubTopicRepository.getTopic("test"), 0); + TopicPartition topicPartition = new TopicPartition("test", 0); + ApacheKafkaOffsetPosition offsetPosition = new ApacheKafkaOffsetPosition(50); + + when(internalKafkaConsumer.assignment()).thenReturn(Collections.emptySet()); + + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, offsetPosition); + assertTrue(kafkaConsumerAdapter.getAssignment().contains(pubSubTopicPartition)); + verify(internalKafkaConsumer).assign(any(List.class)); + verify(internalKafkaConsumer).seek(topicPartition, 51); + } + + @Test + public void testSubscribeTwiceWithSamePartition() { + PubSubTopicPartition pubSubTopicPartition = new PubSubTopicPartitionImpl(pubSubTopicRepository.getTopic("test"), 0); + TopicPartition topicPartition = new TopicPartition("test", 0); + + when(internalKafkaConsumer.assignment()).thenReturn(Collections.emptySet()) + .thenReturn(Collections.singleton(topicPartition)); + + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, 100); + assertTrue(kafkaConsumerAdapter.getAssignment().contains(pubSubTopicPartition)); + kafkaConsumerAdapter.subscribe(pubSubTopicPartition, 200); + verify(internalKafkaConsumer, times(1)).assign(any()); + } + @Test public void testBatchUnsubscribe() { Map topicPartitionBeingUnsubscribedMap = new HashMap<>(); diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java index 6a6280295d4..892bbbc0f41 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/utils/UtilsTest.java @@ -4,11 +4,13 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; import static org.testng.Assert.fail; +import com.linkedin.venice.exceptions.ErrorType; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.exceptions.VeniceHttpException; import com.linkedin.venice.meta.HybridStoreConfig; @@ -82,7 +84,7 @@ public void testParseHostAndPortFromNodeIdentifier() { public void testGetDebugInfo() { Map debugInfo = Utils.getDebugInfo(); debugInfo.forEach((k, v) -> System.out.println(k + ": " + v)); - Assert.assertFalse(debugInfo.isEmpty(), "debugInfo should not be empty."); + assertFalse(debugInfo.isEmpty(), "debugInfo should not be empty."); // N.B.: The "version" entry is not available in unit tests because of the way the classpath is built... String[] expectedKeys = { "path", "host", "pid", "user", "JDK major version" }; assertTrue( @@ -159,8 +161,8 @@ public void testDirectoryExists() throws Exception { Path filePath = Files.createTempFile(null, null); Path nonExistingPath = Paths.get(Utils.getUniqueTempPath()); assertTrue(Utils.directoryExists(directoryPath.toString())); - Assert.assertFalse(Utils.directoryExists(filePath.toString())); - Assert.assertFalse(Utils.directoryExists(nonExistingPath.toString())); + assertFalse(Utils.directoryExists(filePath.toString())); + assertFalse(Utils.directoryExists(nonExistingPath.toString())); Files.delete(directoryPath); Files.delete(filePath); } @@ -434,7 +436,7 @@ public void testParseDateTimeToEpoch() throws Exception { @Test public void testIsSeparateTopicRegion() { Assert.assertTrue(Utils.isSeparateTopicRegion("dc-0_sep")); - Assert.assertFalse(Utils.isSeparateTopicRegion("dc-0")); + assertFalse(Utils.isSeparateTopicRegion("dc-0")); } @Test @@ -521,16 +523,55 @@ public Object[][] booleanParsingData() { }; } + @DataProvider(name = "booleanOrFalseParsingData") + public Object[][] booleanOrFalseParsingData() { + return new Object[][] { + // Valid cases + { "true", "testField", true }, // Valid "true" + { "false", "testField", false }, // Valid "false" + { "TRUE", "testField", true }, // Valid case-insensitive "TRUE" + { "FALSE", "testField", false }, // Valid case-insensitive "FALSE" + { null, "testField", false }, // Null input + + // Invalid cases + { "notABoolean", "testField", null }, // Invalid string + { "123", "testField", null }, // Non-boolean numeric string + { "", "testField", null }, // Empty string + }; + } + @Test(dataProvider = "booleanParsingData") - public void testParseBooleanFromString(String value, String fieldName, Boolean expectedResult) { + public void testParseBooleanOrThrow(String value, String fieldName, Boolean expectedResult) { if (expectedResult != null) { // For valid cases - boolean result = Utils.parseBooleanFromString(value, fieldName); - assertEquals((boolean) expectedResult, result, "Parsed boolean value does not match expected value."); + boolean result = Utils.parseBooleanOrThrow(value, fieldName); + assertEquals(result, (boolean) expectedResult, "Parsed boolean value does not match expected value."); + return; + } + VeniceHttpException e = expectThrows(VeniceHttpException.class, () -> Utils.parseBooleanOrThrow(value, fieldName)); + assertEquals(e.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST, "Invalid status code."); + if (value == null) { + assertEquals(e.getMessage(), "Http Status 400 - testField must be a boolean, but value is null."); + } else { + assertEquals( + e.getMessage(), + "Http Status 400 - testField must be a boolean, but value: " + value + " is invalid."); + } + assertEquals(e.getErrorType(), ErrorType.BAD_REQUEST); + } + + @Test(dataProvider = "booleanOrFalseParsingData") + public void testParseBooleanOrFalse(String value, String fieldName, Boolean expectedResult) { + // For valid cases + if (expectedResult != null) { + boolean result = Utils.parseBooleanOrFalse(value, fieldName); + assertEquals(result, (boolean) expectedResult, "Parsed boolean value does not match expected value."); return; } - VeniceHttpException e = - expectThrows(VeniceHttpException.class, () -> Utils.parseBooleanFromString(value, fieldName)); + // For invalid cases + VeniceHttpException e = expectThrows(VeniceHttpException.class, () -> Utils.parseBooleanOrThrow(value, fieldName)); assertEquals(e.getHttpStatusCode(), HttpStatus.SC_BAD_REQUEST, "Invalid status code."); + assertEquals(e.getMessage(), "Http Status 400 - testField must be a boolean, but value: " + value + " is invalid."); + assertEquals(e.getErrorType(), ErrorType.BAD_REQUEST); } } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java index 2aa63ccfe2d..f7ccf0355dc 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java @@ -14,9 +14,11 @@ import com.linkedin.d2.balancer.D2ClientBuilder; import com.linkedin.davinci.client.DaVinciClient; import com.linkedin.davinci.client.DaVinciConfig; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.StorageClass; import com.linkedin.davinci.client.factory.CachingDaVinciClientFactory; import com.linkedin.venice.D2.D2ClientUtils; +import com.linkedin.venice.endToEnd.TestStringRecordTransformer; import com.linkedin.venice.integration.utils.DaVinciTestContext; import com.linkedin.venice.integration.utils.ServiceFactory; import io.tehuti.metrics.MetricsRepository; @@ -46,6 +48,8 @@ public static void main(String[] args) throws InterruptedException, ExecutionExc int blobTransferServerPort = Integer.parseInt(args[6]); int blobTransferClientPort = Integer.parseInt(args[7]); String storageClass = args[8]; // DISK or MEMORY_BACKED_BY_DISK + boolean recordTransformerEnabled = Boolean.parseBoolean(args[9]); + D2Client d2Client = new D2ClientBuilder().setZkHosts(zkHosts) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) @@ -64,6 +68,16 @@ public static void main(String[] args) throws InterruptedException, ExecutionExc // convert the storage class string to enum StorageClass storageClassEnum = StorageClass.valueOf(storageClass); + DaVinciConfig daVinciConfig = new DaVinciConfig(); + daVinciConfig.setStorageClass(storageClassEnum); + + if (recordTransformerEnabled) { + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); + daVinciConfig.setRecordTransformerConfig(recordTransformerConfig); + } + DaVinciTestContext daVinciTestContext = ServiceFactory.getGenericAvroDaVinciFactoryAndClientWithRetries( d2Client, @@ -71,7 +85,7 @@ public static void main(String[] args) throws InterruptedException, ExecutionExc Optional.empty(), zkHosts, storeName, - new DaVinciConfig().setStorageClass(storageClassEnum), + daVinciConfig, extraBackendConfig); try (CachingDaVinciClientFactory ignored = daVinciTestContext.getDaVinciClientFactory(); DaVinciClient client = daVinciTestContext.getDaVinciClient()) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java index ac5c0002893..331870d3335 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java @@ -9,6 +9,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceMessageException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.kafka.protocol.KafkaMessageEnvelope; import com.linkedin.venice.partitioner.DefaultVenicePartitioner; @@ -103,7 +104,9 @@ public abstract class ConsumerIntegrationTest { @BeforeClass public void sharedSetUp() { - cluster = ServiceFactory.getVeniceCluster(); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = ServiceFactory.getVeniceCluster(options); controllerClient = ControllerClient.constructClusterControllerClient(cluster.getClusterName(), cluster.getAllControllersURLs()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java index 8a52c8ae554..6a84c5352a1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java @@ -35,6 +35,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.integration.utils.ZkServerWrapper; @@ -93,18 +94,18 @@ public void setUp() { serverProperties.put( CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java index 24cb4458c55..81584ae7f01 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java @@ -53,6 +53,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.integration.utils.ZkServerWrapper; @@ -137,18 +138,18 @@ public void setUp() { CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); serverProperties.put(SERVER_AA_WC_WORKLOAD_PARALLEL_PROCESSING_ENABLED, isAAWCParallelProcessingEnabled()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java index fdd30afd693..fb9008b7711 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java @@ -15,6 +15,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.utils.TestUtils; @@ -23,7 +24,6 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -51,18 +51,18 @@ public void setUp() { parentControllerProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "false"); parentControllerProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, "false"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 2, - 2, - 2, - 2, - 1, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.empty(), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(2) + .numberOfChildControllers(2) + .numberOfServers(2) + .numberOfRouters(2) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(parentControllerProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); clusterNames = multiRegionMultiClusterWrapper.getClusterNames(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java index a69bfeb32d2..31e8e18b315 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java @@ -17,6 +17,7 @@ import com.linkedin.venice.helix.HelixReadOnlyLiveClusterConfigRepository; import com.linkedin.venice.helix.ZkClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.Version; @@ -47,7 +48,16 @@ public void setUp() { properties.setProperty(LOCAL_REGION_NAME, "dc-0"); properties.setProperty(ALLOW_CLUSTER_WIPE, "true"); properties.setProperty(TOPIC_CLEANUP_DELAY_FACTOR, "0"); - venice = ServiceFactory.getVeniceCluster(1, 1, 1, 1, 100000, false, false, properties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(100000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(properties) + .build(); + venice = ServiceFactory.getVeniceCluster(options); clusterName = venice.getClusterName(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java index 6bb682135b3..2027db5b454 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java @@ -15,6 +15,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.utils.TestUtils; @@ -56,18 +57,19 @@ public void setUp() { childControllerProperties.setProperty(ALLOW_CLUSTER_WIPE, "true"); Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.of(childControllerProperties), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .childControllerProperties(childControllerProperties) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java index 47e53e776e2..94cee4cae35 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java @@ -18,6 +18,7 @@ import com.linkedin.venice.controllerapi.NewStoreResponse; import com.linkedin.venice.integration.utils.HelixAsAServiceWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.ZkServerWrapper; @@ -68,7 +69,12 @@ public void setUp() { @Test(timeOut = 60 * Time.MS_PER_SECOND) public void testClusterResourceInstanceTag() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { String instanceTag = "GENERAL"; String controllerClusterName = "venice-controllers"; @@ -89,7 +95,12 @@ public void testClusterResourceInstanceTag() { @Test(timeOut = 60 * Time.MS_PER_SECOND) public void testClusterResourceEmptyInstanceTag() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { String instanceTag = ""; String controllerClusterName = "venice-controllers"; @@ -110,7 +121,12 @@ public void testClusterResourceEmptyInstanceTag() { @Test(timeOut = 60 * Time.MS_PER_SECOND) public void testStartHAASHelixControllerAsControllerClusterLeader() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { VeniceControllerWrapper controllerWrapper = venice.addVeniceController(enableControllerClusterHAASProperties); waitForNonDeterministicAssertion( @@ -141,7 +157,12 @@ public void testStartHAASHelixControllerAsControllerClusterLeader() { @Test(timeOut = 120 * Time.MS_PER_SECOND) public void testTransitionToHAASControllerAsControllerClusterLeader() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(3, 1, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(3) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { NewStoreResponse response = venice.getNewStore(Utils.getUniqueString("venice-store")); venice.useControllerClient( @@ -182,7 +203,12 @@ public void testTransitionToHAASControllerAsControllerClusterLeader() { @Test(timeOut = 90 * Time.MS_PER_SECOND) public void testStartHAASControllerAsStorageClusterLeader() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { VeniceControllerWrapper controllerWrapper = venice.addVeniceController(enableControllerAndStorageClusterHAASProperties); @@ -212,7 +238,12 @@ public void testStartHAASControllerAsStorageClusterLeader() { @Test(timeOut = 180 * Time.MS_PER_SECOND) public void testTransitionToHAASControllerAsStorageClusterLeader() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(3, 1, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(3) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { NewStoreResponse response = venice.getNewStore(Utils.getUniqueString("venice-store")); @@ -362,7 +393,12 @@ public void testCloudConfig() { @Test(timeOut = 90 * Time.MS_PER_SECOND) public void testHelixUnknownInstanceOperation() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { VeniceControllerWrapper controllerWrapper = venice.addVeniceController(enableControllerAndStorageClusterHAASProperties); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java index 16fbac94b04..b648d7f1c5f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java @@ -15,6 +15,7 @@ import com.linkedin.venice.helix.HelixCustomizedViewOfflinePushRepository; import com.linkedin.venice.helix.ResourceAssignment; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PartitionAssignment; @@ -42,7 +43,15 @@ public class TestHolisticSeverHealthCheck { @BeforeClass public void setUp() { int numOfController = 1; - cluster = ServiceFactory.getVeniceCluster(numOfController, 2, 1, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(numOfController) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); Properties routerProperties = new Properties(); cluster.addVeniceRouter(routerProperties); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHybridStoreRepartitioningWithMultiDataCenter.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHybridStoreRepartitioningWithMultiDataCenter.java index 8cf6e597dbc..07e4b911e18 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHybridStoreRepartitioningWithMultiDataCenter.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHybridStoreRepartitioningWithMultiDataCenter.java @@ -10,6 +10,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.BackupStrategy; import com.linkedin.venice.meta.StoreInfo; @@ -23,7 +24,6 @@ import com.linkedin.venice.utils.Utils; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -51,17 +51,19 @@ public void setUp() { controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 3); controllerProps.put(DEFAULT_PARTITION_SIZE, 1024); controllerProps.put(CONTROLLER_ENABLE_HYBRID_STORE_PARTITION_COUNT_UPDATE, true); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.empty()); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); topicManagers = new ArrayList<>(2); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java index 7db0ecbe789..1698268f7dd 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java @@ -22,6 +22,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.BackupStrategy; import com.linkedin.venice.meta.BufferReplayPolicy; @@ -45,7 +46,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -80,18 +80,20 @@ public void setUp() { controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 3); controllerProps.put(DEFAULT_PARTITION_SIZE, 1024); Properties serverProps = new Properties(); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProps), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProps); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestStartMultiControllers.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestStartMultiControllers.java index d211d3bfe6e..a9e7e924655 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestStartMultiControllers.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestStartMultiControllers.java @@ -2,6 +2,7 @@ import com.linkedin.venice.helix.SafeHelixManager; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.utils.HelixUtils; @@ -27,7 +28,11 @@ public class TestStartMultiControllers { @BeforeClass public void setUp() throws Exception { - cluster = ServiceFactory.getVeniceCluster(controllerCount, 0, 0); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(controllerCount) + .numberOfServers(0) + .numberOfRouters(0) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); helixManager = new SafeHelixManager( new ZKHelixManager( cluster.getClusterName(), diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestTopicRequestOnHybridDelete.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestTopicRequestOnHybridDelete.java index 70dbe37c5fd..9b94e735620 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestTopicRequestOnHybridDelete.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestTopicRequestOnHybridDelete.java @@ -17,6 +17,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.meta.Version; @@ -48,7 +49,9 @@ public class TestTopicRequestOnHybridDelete { @BeforeClass public void setUp() { - venice = ServiceFactory.getVeniceCluster(); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + venice = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java index b1303d94c23..2efeaa09834 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java @@ -82,7 +82,15 @@ public class VeniceParentHelixAdminTest { @BeforeClass public void setUp() { Utils.thisIsLocalhost(); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(1, 1, 1, 1, 1, 1); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); clusterName = multiRegionMultiClusterWrapper.getClusterNames()[0]; venice = multiRegionMultiClusterWrapper.getChildRegions().get(0).getClusters().get(clusterName); } @@ -239,19 +247,20 @@ public void testResourceCleanupCheckForStoreRecreation() { properties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, String.valueOf(false)); properties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, String.valueOf(false)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(properties) + .childControllerProperties(properties); try ( VeniceTwoLayerMultiRegionMultiClusterWrapper twoLayerMultiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.of(properties), - Optional.of(properties), - Optional.empty()); + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); ControllerClient parentControllerClient = new ControllerClient( twoLayerMultiRegionMultiClusterWrapper.getClusterNames()[0], twoLayerMultiRegionMultiClusterWrapper.getControllerConnectString())) { @@ -433,18 +442,18 @@ public void testSupersetSchemaWithCustomSupersetSchemaGenerator() throws IOExcep properties .put(VeniceControllerWrapper.SUPERSET_SCHEMA_GENERATOR, new SupersetSchemaGeneratorWithCustomProp(CUSTOM_PROP)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(properties); try (VeniceTwoLayerMultiRegionMultiClusterWrapper twoLayerMultiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 0, - 0, - 1, - Optional.of(properties), - Optional.empty(), - Optional.empty())) { + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build())) { String parentControllerUrl = twoLayerMultiRegionMultiClusterWrapper.getControllerConnectString(); try (ControllerClient parentControllerClient = new ControllerClient(twoLayerMultiRegionMultiClusterWrapper.getClusterNames()[0], parentControllerUrl)) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java index 1d52e718f60..f25cc4723ca 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java @@ -9,6 +9,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.DataReplicationPolicy; @@ -49,7 +50,11 @@ public class TestAdminSparkServerWithMultiServers { @BeforeClass public void setUp() { - cluster = ServiceFactory.getVeniceCluster(1, STORAGE_NODE_COUNT, 0); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(STORAGE_NODE_COUNT) + .numberOfRouters(0) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); controllerClient = ControllerClient.constructClusterControllerClient(cluster.getClusterName(), cluster.getAllControllersURLs()); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java index ae4f6685f4c..c41f4cde894 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java @@ -56,6 +56,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; @@ -141,18 +142,20 @@ public void setUp() { Properties controllerProps = new Properties(); controllerProps.put(NATIVE_REPLICATION_SOURCE_FABRIC, "dc-0"); controllerProps.put(PARENT_KAFKA_CLUSTER_FABRIC_LIST, DEFAULT_PARENT_DATA_CENTER_REGION_NAME); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/BlobP2PTransferAmongServersTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/BlobP2PTransferAmongServersTest.java index a997459055e..4ef011f4fc7 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/BlobP2PTransferAmongServersTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/BlobP2PTransferAmongServersTest.java @@ -12,6 +12,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -294,7 +295,12 @@ public VeniceClusterWrapper initializeVeniceCluster(boolean sameRocksDBFormat) { port2 = TestUtils.getFreePort(); } - VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(1, 0, 0, 2); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(2) + .build(); + VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options); // add first server Properties serverProperties = new Properties(); serverProperties.put(ConfigKeys.PERSISTENCE_TYPE, PersistenceType.ROCKS_DB); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/CheckSumTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/CheckSumTest.java index 88a3dd3e383..7bbbb60f47b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/CheckSumTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/CheckSumTest.java @@ -24,6 +24,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.kafka.validation.checksum.CheckSumType; import com.linkedin.venice.meta.PersistenceType; @@ -71,7 +72,16 @@ public void cleanUp() { private VeniceClusterWrapper setUpCluster() { Properties extraProperties = new Properties(); extraProperties.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 1, 1, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); // Add Venice Router Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java index 52fc30e12c9..24352023178 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java @@ -1,18 +1,24 @@ package com.linkedin.venice.endToEnd; +import static com.linkedin.davinci.store.rocksdb.RocksDBServerConfig.ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED; +import static com.linkedin.venice.ConfigKeys.BLOB_TRANSFER_MANAGER_ENABLED; import static com.linkedin.venice.ConfigKeys.CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS; import static com.linkedin.venice.ConfigKeys.CLIENT_USE_SYSTEM_STORE_REPOSITORY; import static com.linkedin.venice.ConfigKeys.DATA_BASE_PATH; +import static com.linkedin.venice.ConfigKeys.DAVINCI_P2P_BLOB_TRANSFER_CLIENT_PORT; +import static com.linkedin.venice.ConfigKeys.DAVINCI_P2P_BLOB_TRANSFER_SERVER_PORT; import static com.linkedin.venice.ConfigKeys.DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS; import static com.linkedin.venice.ConfigKeys.DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS; import static com.linkedin.venice.ConfigKeys.DA_VINCI_CURRENT_VERSION_BOOTSTRAPPING_SPEEDUP_ENABLED; import static com.linkedin.venice.ConfigKeys.PERSISTENCE_TYPE; import static com.linkedin.venice.ConfigKeys.PUSH_STATUS_STORE_ENABLED; +import static com.linkedin.venice.ConfigKeys.SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE; import static com.linkedin.venice.ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS; import static com.linkedin.venice.integration.utils.VeniceClusterWrapper.DEFAULT_KEY_SCHEMA; import static com.linkedin.venice.meta.PersistenceType.ROCKS_DB; import static com.linkedin.venice.utils.IntegrationTestPushUtils.createStoreForJob; import static com.linkedin.venice.utils.IntegrationTestPushUtils.defaultVPJProps; +import static com.linkedin.venice.utils.TestWriteUtils.DEFAULT_USER_DATA_RECORD_COUNT; import static com.linkedin.venice.utils.TestWriteUtils.getTempDataDirectory; import static com.linkedin.venice.utils.TestWriteUtils.writeSimpleAvroFileWithIntToIntSchema; import static com.linkedin.venice.utils.TestWriteUtils.writeSimpleAvroFileWithIntToStringSchema; @@ -23,6 +29,7 @@ import com.linkedin.d2.balancer.D2Client; import com.linkedin.d2.balancer.D2ClientBuilder; +import com.linkedin.davinci.DaVinciUserApp; import com.linkedin.davinci.client.DaVinciClient; import com.linkedin.davinci.client.DaVinciConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; @@ -33,9 +40,12 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; +import com.linkedin.venice.store.rocksdb.RocksDBUtils; import com.linkedin.venice.utils.DataProviderUtils; +import com.linkedin.venice.utils.ForkedJavaProcess; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.TestWriteUtils; @@ -44,15 +54,17 @@ import io.tehuti.metrics.MetricsRepository; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import org.apache.avro.Schema; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -69,7 +81,16 @@ public void setUp() { clusterConfig.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); clusterConfig.put(PUSH_STATUS_STORE_ENABLED, true); clusterConfig.put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 3); - cluster = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); d2Client = new D2ClientBuilder().setZkHosts(cluster.getZk().getAddress()) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) @@ -85,16 +106,6 @@ public void cleanUp() { Utils.closeQuietlyWithErrorLogged(cluster); } - @BeforeMethod - @AfterClass - public void deleteClassHash() { - int storeVersion = 1; - File file = new File(String.format("./classHash-%d.txt", storeVersion)); - if (file.exists()) { - assertTrue(file.delete()); - } - } - @Test(timeOut = TEST_TIMEOUT, dataProvider = "dv-client-config-provider", dataProviderClass = DataProviderUtils.class) public void testRecordTransformer(DaVinciConfig clientConfig) throws Exception { String storeName = Utils.getUniqueString("test-store"); @@ -115,15 +126,9 @@ public void testRecordTransformer(DaVinciConfig clientConfig) throws Exception { metricsRepository, backendConfig)) { - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new TestStringRecordTransformer( - storeVersion, - keySchema, - inputValueSchema, - outputValueSchema, - true), - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); clientConfig.setRecordTransformerConfig(recordTransformerConfig); DaVinciClient clientWithRecordTransformer = @@ -161,15 +166,12 @@ public void testTypeChangeRecordTransformer(DaVinciConfig clientConfig) throws E VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME, metricsRepository, backendConfig)) { - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new TestIntToStringRecordTransformer( - storeVersion, - keySchema, - inputValueSchema, - outputValueSchema, - true), - String.class, - Schema.create(Schema.Type.STRING)); + + DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction(TestIntToStringRecordTransformer::new) + .setOutputValueClass(String.class) + .setOutputValueSchema(Schema.create(Schema.Type.STRING)) + .build(); clientConfig.setRecordTransformerConfig(recordTransformerConfig); DaVinciClient clientWithRecordTransformer = @@ -213,13 +215,18 @@ public void testRecordTransformerOnRecovery(DaVinciConfig clientConfig) throws E Schema myKeySchema = Schema.create(Schema.Type.INT); Schema myValueSchema = Schema.create(Schema.Type.STRING); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); + TestStringRecordTransformer recordTransformer = - new TestStringRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, true); + new TestStringRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, dummyRecordTransformerConfig); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> recordTransformer, - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction( + (storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> recordTransformer) + .build(); clientConfig.setRecordTransformerConfig(recordTransformerConfig); DaVinciClient clientWithRecordTransformer = @@ -292,13 +299,18 @@ public void testRecordTransformerChunking(DaVinciConfig clientConfig) throws Exc Schema myKeySchema = Schema.create(Schema.Type.INT); Schema myValueSchema = Schema.create(Schema.Type.STRING); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); + TestStringRecordTransformer recordTransformer = - new TestStringRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, true); + new TestStringRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, dummyRecordTransformerConfig); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> recordTransformer, - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction( + (storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> recordTransformer) + .build(); clientConfig.setRecordTransformerConfig(recordTransformerConfig); DaVinciClient clientWithRecordTransformer = @@ -356,13 +368,19 @@ public void testRecordTransformerWithEmptyDaVinci(DaVinciConfig clientConfig) th Schema myKeySchema = Schema.create(Schema.Type.INT); Schema myValueSchema = Schema.create(Schema.Type.STRING); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .setStoreRecordsInDaVinci(false) + .build(); + TestStringRecordTransformer recordTransformer = - new TestStringRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, false); + new TestStringRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, dummyRecordTransformerConfig); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> recordTransformer, - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction( + (storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> recordTransformer) + .setStoreRecordsInDaVinci(false) + .build(); clientConfig.setRecordTransformerConfig(recordTransformerConfig); try (CachingDaVinciClientFactory factory = new CachingDaVinciClientFactory( @@ -405,13 +423,18 @@ public void testSkipResultRecordTransformer(DaVinciConfig clientConfig) throws E Schema myKeySchema = Schema.create(Schema.Type.INT); Schema myValueSchema = Schema.create(Schema.Type.STRING); + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestSkipResultRecordTransformer::new) + .build(); + TestSkipResultRecordTransformer recordTransformer = - new TestSkipResultRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, true); + new TestSkipResultRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, dummyRecordTransformerConfig); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> recordTransformer, - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction( + (storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> recordTransformer) + .build(); clientConfig.setRecordTransformerConfig(recordTransformerConfig); try (CachingDaVinciClientFactory factory = new CachingDaVinciClientFactory( @@ -451,15 +474,9 @@ public void testUnchangedResultRecordTransformer(DaVinciConfig clientConfig) thr VeniceProperties backendConfig = buildRecordTransformerBackendConfig(pushStatusStoreEnabled); MetricsRepository metricsRepository = new MetricsRepository(); - DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig( - (storeVersion, keySchema, inputValueSchema, outputValueSchema) -> new TestUnchangedResultRecordTransformer( - storeVersion, - keySchema, - inputValueSchema, - outputValueSchema, - true), - String.class, - Schema.create(Schema.Type.STRING)); + DaVinciRecordTransformerConfig recordTransformerConfig = new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction(TestUnchangedResultRecordTransformer::new) + .build(); clientConfig.setRecordTransformerConfig(recordTransformerConfig); try (CachingDaVinciClientFactory factory = new CachingDaVinciClientFactory( @@ -486,6 +503,101 @@ public void testUnchangedResultRecordTransformer(DaVinciConfig clientConfig) thr } } + @Test(timeOut = 2 * TEST_TIMEOUT) + public void testBlobTransferRecordTransformer() throws Exception { + String dvcPath1 = Utils.getTempDataDirectory().getAbsolutePath(); + boolean pushStatusStoreEnabled = true; + String zkHosts = cluster.getZk().getAddress(); + int port1 = TestUtils.getFreePort(); + int port2 = TestUtils.getFreePort(); + while (port1 == port2) { + port2 = TestUtils.getFreePort(); + } + Consumer paramsConsumer = params -> params.setBlobTransferEnabled(true); + String storeName = Utils.getUniqueString("test-store"); + DaVinciConfig clientConfig = new DaVinciConfig(); + + Schema myKeySchema = Schema.create(Schema.Type.INT); + Schema myValueSchema = Schema.create(Schema.Type.STRING); + + DaVinciRecordTransformerConfig dummyRecordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder().setRecordTransformerFunction(TestStringRecordTransformer::new) + .build(); + + TestStringRecordTransformer recordTransformer = + new TestStringRecordTransformer(1, myKeySchema, myValueSchema, myValueSchema, dummyRecordTransformerConfig); + + DaVinciRecordTransformerConfig recordTransformerConfig = + new DaVinciRecordTransformerConfig.Builder() + .setRecordTransformerFunction( + (storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> recordTransformer) + .build(); + clientConfig.setRecordTransformerConfig(recordTransformerConfig); + + setUpStore(storeName, paramsConsumer, properties -> {}, pushStatusStoreEnabled); + LOGGER.info("Port1 is {}, Port2 is {}", port1, port2); + LOGGER.info("zkHosts is {}", zkHosts); + + // Start the first DaVinci Client using DaVinciUserApp for regular ingestion + ForkedJavaProcess.exec( + DaVinciUserApp.class, + zkHosts, + dvcPath1, + storeName, + "100", + "10", + "false", + Integer.toString(port1), + Integer.toString(port2), + StorageClass.DISK.toString(), + "true"); + + // Wait for the first DaVinci Client to complete ingestion + Thread.sleep(60000); + + // Start the second DaVinci Client using settings for blob transfer + String dvcPath2 = Utils.getTempDataDirectory().getAbsolutePath(); + + PropertyBuilder configBuilder = new PropertyBuilder().put(PERSISTENCE_TYPE, ROCKS_DB) + .put(ROCKSDB_PLAIN_TABLE_FORMAT_ENABLED, "false") + .put(SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE, "3000") + .put(CLIENT_USE_SYSTEM_STORE_REPOSITORY, true) + .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) + .put(DATA_BASE_PATH, dvcPath2) + .put(DAVINCI_P2P_BLOB_TRANSFER_SERVER_PORT, port2) + .put(DAVINCI_P2P_BLOB_TRANSFER_CLIENT_PORT, port1) + .put(PUSH_STATUS_STORE_ENABLED, true) + .put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 1) + .put(BLOB_TRANSFER_MANAGER_ENABLED, true); + VeniceProperties backendConfig2 = configBuilder.build(); + + try (CachingDaVinciClientFactory factory2 = new CachingDaVinciClientFactory( + d2Client, + VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME, + new MetricsRepository(), + backendConfig2)) { + DaVinciClient client2 = factory2.getAndStartGenericAvroClient(storeName, clientConfig); + client2.subscribeAll().get(); + + // Verify snapshots exists + for (int i = 0; i < 3; i++) { + String snapshotPath = RocksDBUtils.composeSnapshotDir(dvcPath1 + "/rocksdb", storeName + "_v1", i); + Assert.assertTrue(Files.exists(Paths.get(snapshotPath))); + } + + // All of the records should have already been transformed due to blob transfer + assertEquals(recordTransformer.getTransformInvocationCount(), 0); + + // Test single-get access + for (int k = 1; k <= DEFAULT_USER_DATA_RECORD_COUNT; ++k) { + Object valueObj = client2.get(k).get(); + String expectedValue = "name " + k + "Transformed"; + assertEquals(valueObj.toString(), expectedValue); + assertEquals(recordTransformer.get(k), expectedValue); + } + } + } + /* * Batch data schema: * Key: Integer @@ -523,6 +635,41 @@ protected void setUpStore( inputDir); } + /* + * Batch data schema: + * Key: Integer + * Value: String + */ + private void setUpStore( + String storeName, + Consumer paramsConsumer, + Consumer propertiesConsumer, + boolean useDVCPushStatusStore) { + boolean chunkingEnabled = false; + CompressionStrategy compressionStrategy = CompressionStrategy.NO_OP; + + File inputDir = getTempDataDirectory(); + + Runnable writeAvroFileRunnable = () -> { + try { + writeSimpleAvroFileWithIntToStringSchema(inputDir); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + String valueSchema = "\"string\""; + setUpStore( + storeName, + paramsConsumer, + propertiesConsumer, + useDVCPushStatusStore, + chunkingEnabled, + compressionStrategy, + writeAvroFileRunnable, + valueSchema, + inputDir); + } + protected void setUpStore( String storeName, Consumer paramsConsumer, @@ -609,9 +756,7 @@ public VeniceProperties buildRecordTransformerBackendConfig(boolean pushStatusSt .put(CLIENT_SYSTEM_STORE_REPOSITORY_REFRESH_INTERVAL_SECONDS, 1) .put(DATA_BASE_PATH, baseDataPath) .put(PERSISTENCE_TYPE, ROCKS_DB) - .put(DA_VINCI_CURRENT_VERSION_BOOTSTRAPPING_SPEEDUP_ENABLED, true) - .put(PUSH_STATUS_STORE_ENABLED, pushStatusStoreEnabled) - .put(DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS, 1000); + .put(DA_VINCI_CURRENT_VERSION_BOOTSTRAPPING_SPEEDUP_ENABLED, true); if (pushStatusStoreEnabled) { backendPropertyBuilder.put(PUSH_STATUS_STORE_ENABLED, true).put(DAVINCI_PUSH_STATUS_CHECK_INTERVAL_IN_MS, 1000); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java index e37b7bf4729..d59b2503d68 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java @@ -70,6 +70,7 @@ import com.linkedin.venice.ingestion.protocol.IngestionStorageMetadata; import com.linkedin.venice.integration.utils.DaVinciTestContext; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.IngestionMetadataUpdateType; @@ -131,7 +132,6 @@ import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -153,7 +153,16 @@ public void setUp() { clusterConfig.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); clusterConfig.put(PUSH_STATUS_STORE_ENABLED, true); clusterConfig.put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 3); - cluster = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); d2Client = new D2ClientBuilder().setZkHosts(cluster.getZk().getAddress()) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) @@ -171,16 +180,6 @@ public void cleanUp() { Utils.closeQuietlyWithErrorLogged(cluster); } - @BeforeMethod - @AfterClass - public void deleteClassHash() { - int storeVersion = 1; - File file = new File(String.format("./classHash-%d.txt", storeVersion)); - if (file.exists()) { - assertTrue(file.delete()); - } - } - @Test(timeOut = TEST_TIMEOUT) public void testConcurrentGetAndStart() throws Exception { String s1 = createStoreWithMetaSystemStoreAndPushStatusSystemStore(KEY_COUNT); @@ -1098,7 +1097,8 @@ public void testCrashedDaVinciWithIngestionIsolation() throws Exception { "true", Integer.toString(port1), Integer.toString(port2), - StorageClass.DISK.toString()); + StorageClass.DISK.toString(), + "false"); // Sleep long enough so the forked Da Vinci app process can finish ingestion. Thread.sleep(60000); IsolatedIngestionUtils.executeShellCommand("kill " + forkedDaVinciUserApp.pid()); @@ -1173,7 +1173,8 @@ public void testBlobP2PTransferAmongDVC() throws Exception { "false", Integer.toString(port1), Integer.toString(port2), - StorageClass.DISK.toString()); + StorageClass.DISK.toString(), + "false"); // Wait for the first DaVinci Client to complete ingestion Thread.sleep(60000); @@ -1208,6 +1209,13 @@ public void testBlobP2PTransferAmongDVC() throws Exception { String snapshotPath = RocksDBUtils.composeSnapshotDir(dvcPath1 + "/rocksdb", storeName + "_v1", i); Assert.assertTrue(Files.exists(Paths.get(snapshotPath))); } + + for (int i = 0; i < 3; i++) { + String partitionPath2 = RocksDBUtils.composePartitionDbDir(dvcPath2 + "/rocksdb", storeName + "_v1", i); + Assert.assertTrue(Files.exists(Paths.get(partitionPath2))); + String snapshotPath2 = RocksDBUtils.composeSnapshotDir(dvcPath2 + "/rocksdb", storeName + "_v1", i); + Assert.assertFalse(Files.exists(Paths.get(snapshotPath2))); + } } } @@ -1241,7 +1249,8 @@ public void testDVCSnapshotGeneration(boolean useDiskStorage) throws Exception { "false", Integer.toString(port1), Integer.toString(port2), - storageClass); + storageClass, + "false"); // Wait for the first DaVinci Client to complete ingestion Thread.sleep(60000); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java index d8f87545bc4..c82e5a30fdc 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java @@ -25,6 +25,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -40,7 +41,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -78,18 +78,18 @@ public void setUp() { Utils.thisIsLocalhost(); Properties parentControllerProps = new Properties(); parentControllerProps.put(OFFLINE_JOB_START_TIMEOUT_MS, "180000"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 2, - 1, - 1, - 3, - 1, - 3, - Optional.of(parentControllerProps), - Optional.empty(), - Optional.empty(), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(2) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(3) + .numberOfRouters(1) + .replicationFactor(3) + .forkServer(false) + .parentControllerProperties(parentControllerProps); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); multiClusterVenice = multiRegionMultiClusterWrapper.getChildRegions().get(0); clusterNames = multiClusterVenice.getClusterNames(); parentControllerURLs = multiRegionMultiClusterWrapper.getParentControllers() diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java index a15449e50ff..67e6b361a9b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java @@ -29,6 +29,7 @@ import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.DaVinciTestContext; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -141,7 +142,16 @@ public void setUp() { Utils.thisIsLocalhost(); Properties clusterConfig = new Properties(); clusterConfig.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - cluster = ServiceFactory.getVeniceCluster(1, 2, 1, 1, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); d2Client = new D2ClientBuilder().setZkHosts(cluster.getZk().getAddress()) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciLiveUpdateSuppressionTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciLiveUpdateSuppressionTest.java index 3ffd4d21267..7c14a962cb0 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciLiveUpdateSuppressionTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciLiveUpdateSuppressionTest.java @@ -20,6 +20,7 @@ import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.DaVinciTestContext; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.IngestionMode; import com.linkedin.venice.meta.StoreInfo; @@ -66,7 +67,16 @@ public void setUp() { Utils.thisIsLocalhost(); Properties clusterConfig = new Properties(); clusterConfig.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - cluster = ServiceFactory.getVeniceCluster(1, 2, 1, 1, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); d2Client = new D2ClientBuilder().setZkHosts(cluster.getZk().getAddress()) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DataRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DataRecoveryTest.java index 89f0a0bc775..6eb358ff546 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DataRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DataRecoveryTest.java @@ -40,6 +40,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.Version; @@ -98,18 +99,20 @@ public void setUp() { controllerProps.put(ALLOW_CLUSTER_WIPE, "true"); controllerProps.put(TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS, "1000"); controllerProps.put(MIN_NUMBER_OF_UNUSED_KAFKA_TOPICS_TO_PRESERVE, "0"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); clusterName = CLUSTER_NAMES[0]; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/NearlineE2ELatencyTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/NearlineE2ELatencyTest.java index f60a2eb93d9..4fb6674b2f1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/NearlineE2ELatencyTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/NearlineE2ELatencyTest.java @@ -20,6 +20,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -70,18 +71,18 @@ public void setUp() { Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - NUMBER_OF_PARENT_CONTROLLERS, - NUMBER_OF_CONTROLLERS, - NUMBER_OF_SERVERS, - NUMBER_OF_ROUTERS, - REPLICATION_FACTOR, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(NUMBER_OF_PARENT_CONTROLLERS) + .numberOfChildControllers(NUMBER_OF_CONTROLLERS) + .numberOfServers(NUMBER_OF_SERVERS) + .numberOfRouters(NUMBER_OF_ROUTERS) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); multiRegionMultiClusterWrapper.logMultiCluster(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OneTouchDataRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OneTouchDataRecoveryTest.java index e4f80c5d116..b1b6c44311c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OneTouchDataRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OneTouchDataRecoveryTest.java @@ -13,6 +13,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.RegionPushDetails; import com.linkedin.venice.meta.Version; @@ -58,18 +59,18 @@ public void setUp() { Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - NUMBER_OF_PARENT_CONTROLLERS, - NUMBER_OF_CONTROLLERS, - NUMBER_OF_SERVERS, - NUMBER_OF_ROUTERS, - REPLICATION_FACTOR, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(NUMBER_OF_PARENT_CONTROLLERS) + .numberOfChildControllers(NUMBER_OF_CONTROLLERS) + .numberOfServers(NUMBER_OF_SERVERS) + .numberOfRouters(NUMBER_OF_ROUTERS) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateClusterConfigTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateClusterConfigTest.java index 93d3e0cb06e..d3806827051 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateClusterConfigTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateClusterConfigTest.java @@ -14,6 +14,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreInfo; @@ -21,7 +22,6 @@ import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.testng.Assert; @@ -46,18 +46,20 @@ public void setUp() { controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); controllerProps.put(ConfigKeys.ENABLE_PARTIAL_UPDATE_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES, true); controllerProps.put(ConfigKeys.ENABLE_PARTIAL_UPDATE_FOR_HYBRID_NON_ACTIVE_ACTIVE_USER_STORES, true); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java index ab31cccda4f..6303037666c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java @@ -74,6 +74,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.BackupStrategy; @@ -115,7 +116,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -175,18 +175,20 @@ public void setUp() { Properties controllerProps = new Properties(); controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); controllerProps.put(ConfigKeys.PARTICIPANT_MESSAGE_STORE_ENABLED, false); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - REPLICATION_FACTOR, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java index 078a5a51631..7bc8c35ca85 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java @@ -39,6 +39,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.status.PushJobDetailsStatus; @@ -58,7 +59,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.apache.avro.Schema; @@ -100,18 +100,19 @@ private void setUp(boolean useCustomCheckpoints) throws IOException { // Need to add this in controller props when creating venice system for tests parentControllerProperties.setProperty(ConfigKeys.PUSH_JOB_STATUS_STORE_CLUSTER_NAME, "venice-cluster0"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(parentControllerProperties) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); String clusterName = multiRegionMultiClusterWrapper.getClusterNames()[0]; VeniceMultiClusterWrapper childRegionMultiClusterWrapper = multiRegionMultiClusterWrapper.getChildRegions().get(0); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java index 1ebca163050..ad8a8f4fb86 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java @@ -25,6 +25,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; @@ -34,7 +35,6 @@ import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.testng.annotations.AfterClass; @@ -76,18 +76,19 @@ public void setUp() { extraProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, String.valueOf(true)); extraProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, String.valueOf(true)); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - NUMBER_OF_PARENT_CONTROLLERS, - NUMBER_OF_CHILD_CONTROLLERS, - NUMBER_OF_SERVERS, - NUMBER_OF_ROUTERS, - REPLICATION_FACTOR, - Optional.of(extraProperties), - Optional.of(extraProperties), - Optional.empty(), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(NUMBER_OF_PARENT_CONTROLLERS) + .numberOfChildControllers(NUMBER_OF_CHILD_CONTROLLERS) + .numberOfServers(NUMBER_OF_SERVERS) + .numberOfRouters(NUMBER_OF_ROUTERS) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .parentControllerProperties(extraProperties) + .childControllerProperties(extraProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/RocksDBPlainTableTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/RocksDBPlainTableTest.java index 4828c30af99..7cdf1d51142 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/RocksDBPlainTableTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/RocksDBPlainTableTest.java @@ -9,6 +9,7 @@ import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.utils.Time; @@ -25,7 +26,9 @@ public class RocksDBPlainTableTest { @BeforeClass public void setUp() { - veniceCluster = ServiceFactory.getVeniceCluster(1, 0, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(1).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java index fe24d002e2d..7a8cd0e5ad6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java @@ -21,6 +21,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.utils.TestUtils; @@ -62,18 +63,18 @@ public void setUp() { serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); // 1 parent controller, 1 child region, 1 clusters per child region, 2 servers per cluster - twoLayerClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 2, - 0, - 2, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(0) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(parentControllerProperties) + .serverProperties(serverProperties); + twoLayerClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); multiClusterWrapper = twoLayerClusterWrapper.getChildRegions().get(0); String[] clusterNames = multiClusterWrapper.getClusterNames(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveIngestion.java index a92865751ac..c435f74c7ef 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveIngestion.java @@ -37,6 +37,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.VeniceUserStoreType; @@ -126,18 +127,20 @@ public void setUp() { serverProperties.put(SERVER_AA_WC_WORKLOAD_PARALLEL_PROCESSING_ENABLED, isAAWCParallelProcessingEnabled()); Properties controllerProps = new Properties(); controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 20); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java index 63fb50cbaf5..fc53acfe2c8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java @@ -30,6 +30,7 @@ import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.HybridStoreConfig; @@ -103,18 +104,20 @@ public void setUp() { controllerProps.put(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, true); controllerProps.put(ENABLE_INCREMENTAL_PUSH_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES, true); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); clusterNames = multiRegionMultiClusterWrapper.getClusterNames(); clusterName = this.clusterNames[0]; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDelayedLeaderPromotion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDelayedLeaderPromotion.java index d3f50175547..aba05953103 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDelayedLeaderPromotion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDelayedLeaderPromotion.java @@ -22,6 +22,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.VeniceUserStoreType; @@ -62,18 +63,18 @@ public void setUp() { serverProperties.put( CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java index 534490697e8..93bd5c4ffda 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java @@ -22,6 +22,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.samza.VeniceSystemProducer; @@ -87,18 +88,20 @@ public void setUp() { controllerProps.put(NATIVE_REPLICATION_SOURCE_FABRIC, "dc-0"); controllerProps.put(PARENT_KAFKA_CLUSTER_FABRIC_LIST, DEFAULT_PARENT_DATA_CENTER_REGION_NAME); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBackupVersionDatabaseOptimization.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBackupVersionDatabaseOptimization.java index 9b3ab2ff9cb..a803a10f324 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBackupVersionDatabaseOptimization.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBackupVersionDatabaseOptimization.java @@ -20,6 +20,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.utils.TestUtils; @@ -52,7 +53,16 @@ public void setUp() { extraProperties.setProperty(SERVER_OPTIMIZE_DATABASE_SERVICE_SCHEDULE_INTERNAL_SECONDS, "1"); extraProperties.setProperty(SERVER_OPTIMIZE_DATABASE_FOR_BACKUP_VERSION_NO_READ_THRESHOLD_SECONDS, "3"); - venice = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + venice = ServiceFactory.getVeniceCluster(options); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java index 4015671b616..61852c7ae47 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java @@ -11,6 +11,7 @@ import static com.linkedin.venice.ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; import java.util.Properties; @@ -21,7 +22,9 @@ public class TestBatchForRocksDB extends TestBatch { @Override public VeniceClusterWrapper initializeVeniceCluster() { - VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(1, 0, 0); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(0).build(); + VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options); Properties serverProperties = new Properties(); serverProperties.put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java index cef9e24a1e5..2e01a0a0f1a 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java @@ -38,6 +38,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -53,7 +54,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -79,18 +79,20 @@ public void setUp() { serverProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1L)); serverProperties.setProperty(SERVER_BATCH_REPORT_END_OF_INCREMENTAL_PUSH_STATUS_ENABLED, Long.toString(60L)); Properties controllerProps = new Properties(); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwap.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwap.java index b71eafc8f8e..cdc9d3edaa9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwap.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwap.java @@ -10,6 +10,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.meta.Version; @@ -21,7 +22,6 @@ import java.io.File; import java.io.IOException; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -47,17 +47,20 @@ public void setUp() { controllerProps.put(CONTROLLER_DEFERRED_VERSION_SWAP_SERVICE_ENABLED, true); Properties serverProperties = new Properties(); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java index a5a67cd9fc0..4a461e63d11 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java @@ -20,6 +20,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; @@ -28,7 +29,6 @@ import com.linkedin.venice.utils.Utils; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.apache.avro.Schema; @@ -62,18 +62,20 @@ public void setUp() { KafkaConsumerServiceDelegator.ConsumerPoolStrategyType.CURRENT_VERSION_PRIORITIZATION.name()); Properties controllerProps = new Properties(); controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - REPLICATION_FACTOR, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); this.parentController = parentControllers.get(0); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestFatalDataValidationExceptionHandling.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestFatalDataValidationExceptionHandling.java index cdd919d02cd..377acd03fc2 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestFatalDataValidationExceptionHandling.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestFatalDataValidationExceptionHandling.java @@ -37,6 +37,7 @@ import com.linkedin.venice.guid.GuidUtils; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.meta.Version; @@ -94,7 +95,16 @@ private VeniceClusterWrapper setUpCluster() { extraProperties .setProperty(FATAL_DATA_VALIDATION_FAILURE_TOPIC_RETENTION_MS, Long.toString(TimeUnit.SECONDS.toMillis(30))); extraProperties.setProperty(MIN_NUMBER_OF_UNUSED_KAFKA_TOPICS_TO_PRESERVE, "0"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 1, 1, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); // Add Venice Router Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHelixCustomizedView.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHelixCustomizedView.java index e5a85896c08..507f4d3f1da 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHelixCustomizedView.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHelixCustomizedView.java @@ -11,6 +11,7 @@ import com.linkedin.venice.helix.VeniceOfflinePushMonitorAccessor; import com.linkedin.venice.helix.ZkClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -80,7 +81,15 @@ public class TestHelixCustomizedView { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 0, 0, replicationFactor, 10, true, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(replicationFactor) + .partitionSize(10) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); admin = new ZKHelixAdmin(veniceCluster.getZk().getAddress()); Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java index 9cf804fd5ea..f4a22b119e0 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java @@ -56,6 +56,7 @@ import com.linkedin.venice.helix.HelixBaseRoutingRepository; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.kafka.protocol.GUID; @@ -475,8 +476,16 @@ public void testSamzaBatchLoad() throws Exception { extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1L)); SystemProducer veniceBatchProducer = null; - try (VeniceClusterWrapper veniceClusterWrapper = - ServiceFactory.getVeniceCluster(1, 3, 1, 2, 1000000, false, false, extraProperties)) { + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(3) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + try (VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options)) { try { Admin admin = veniceClusterWrapper.getLeaderVeniceController().getVeniceAdmin(); String clusterName = veniceClusterWrapper.getClusterName(); @@ -725,9 +734,16 @@ public void testMultiStreamReprocessingSystemProducers() { public void testLeaderHonorLastTopicSwitchMessage() throws Exception { Properties extraProperties = new Properties(); extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(10L)); - try ( - VeniceClusterWrapper venice = - ServiceFactory.getVeniceCluster(1, 2, 1, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); ControllerClient controllerClient = new ControllerClient(venice.getClusterName(), venice.getAllControllersURLs())) { long streamingRewindSeconds = 25L; @@ -1606,8 +1622,16 @@ public void testLeaderShouldCalculateRewindDuringPromotion() { extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(20L)); final int partitionCount = 1; final int keyCount = 10; - try (VeniceClusterWrapper cluster = - ServiceFactory.getVeniceCluster(1, 1, 1, 1, 1000000, false, false, extraProperties)) { + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { UpdateStoreQueryParams params = new UpdateStoreQueryParams() // set hybridRewindSecond to a big number so following versions won't ignore old records in RT .setHybridRewindSeconds(10) @@ -1695,7 +1719,16 @@ private static Pair getKafkaKeyAndValueEnvelope( private static VeniceClusterWrapper setUpCluster(boolean enablePartitionWiseSharedConsumer) { Properties extraProperties = new Properties(); extraProperties.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 0, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); // Add Venice Router Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java index 9d9fae8396b..daca19c9a4c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java @@ -28,6 +28,7 @@ import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.HybridStoreConfig; @@ -339,19 +340,20 @@ private static VeniceTwoLayerMultiRegionMultiClusterWrapper setUpCluster() { serverProperties.setProperty(SERVER_CONSUMER_POOL_SIZE_PER_KAFKA_CLUSTER, "3"); serverProperties.setProperty(SERVER_DEDICATED_DRAINER_FOR_SORTED_INPUT_ENABLED, "true"); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(Optional.of(parentControllerProps).orElse(null)) + .childControllerProperties(Optional.of(childControllerProperties).orElse(null)) + .serverProperties(serverProperties); VeniceTwoLayerMultiRegionMultiClusterWrapper cluster = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 2, - 1, - 1, - Optional.of(parentControllerProps), - Optional.of(childControllerProperties), - Optional.of(serverProperties), - false); + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); return cluster; } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridQuota.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridQuota.java index e573d112f5d..763f9853342 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridQuota.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridQuota.java @@ -28,6 +28,7 @@ import com.linkedin.venice.helix.SafeHelixManager; import com.linkedin.venice.helix.ZkClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.meta.Store; @@ -73,7 +74,16 @@ public void setUp() { extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1L)); // N.B.: RF 2 with 3 servers is important, in order to test both the leader and follower code paths - sharedVenice = ServiceFactory.getVeniceCluster(1, 0, 0, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + sharedVenice = ServiceFactory.getVeniceCluster(options); Properties routerProperties = new Properties(); routerProperties.put(HELIX_HYBRID_STORE_QUOTA_ENABLED, true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestIntToStringRecordTransformer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestIntToStringRecordTransformer.java index 18ad52ec13d..76a067322ae 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestIntToStringRecordTransformer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestIntToStringRecordTransformer.java @@ -1,6 +1,7 @@ package com.linkedin.venice.endToEnd; import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.venice.utils.lazy.Lazy; import java.io.IOException; @@ -16,8 +17,8 @@ public TestIntToStringRecordTransformer( Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci) { - super(storeVersion, keySchema, inputValueSchema, outputValueSchema, storeRecordsInDaVinci); + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); } @Override diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestLeaderReplicaFailover.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestLeaderReplicaFailover.java index 31fa014783b..01de56380e5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestLeaderReplicaFailover.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestLeaderReplicaFailover.java @@ -23,6 +23,7 @@ import com.linkedin.venice.helix.HelixBaseRoutingRepository; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.Instance; @@ -92,8 +93,17 @@ public void setUp() { int numberOfServer = 3; int numberOfRouter = 1; - clusterWrapper = ServiceFactory - .getVeniceCluster(numberOfController, numberOfServer, numberOfRouter, 3, 1, false, false, serverProperties); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(numberOfServer) + .numberOfRouters(numberOfRouter) + .replicationFactor(3) + .partitionSize(1) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(serverProperties) + .build(); + clusterWrapper = ServiceFactory.getVeniceCluster(options); clusterName = clusterWrapper.getClusterName(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java index 2e30a113a01..2f0fd279ad0 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java @@ -30,6 +30,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.MaterializedViewParameters; @@ -54,7 +55,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -84,18 +84,18 @@ public void setUp() { serverProperties.put( CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 2, - 1, - 1, - 1, - 2, - 1, - 2, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(2) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMultiDataCenterAdminOperations.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMultiDataCenterAdminOperations.java index 67fd2815614..d469b2c726f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMultiDataCenterAdminOperations.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMultiDataCenterAdminOperations.java @@ -16,6 +16,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; @@ -26,7 +27,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -60,18 +60,18 @@ public class TestMultiDataCenterAdminOperations { public void setUp() { Properties serverProperties = new Properties(); serverProperties.setProperty(ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1)); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childClusters = multiRegionMultiClusterWrapper.getChildRegions(); childControllers = childClusters.stream() diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java index aa60a5116cb..1f93b67b518 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java @@ -24,6 +24,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.samza.VeniceObjectWithTimestamp; @@ -42,7 +43,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -112,18 +112,20 @@ public void setUp() throws IOException { controllerProps.put(NATIVE_REPLICATION_SOURCE_FABRIC, "dc-0"); controllerProps.put(PARENT_KAFKA_CLUSTER_FABRIC_LIST, DEFAULT_PARENT_DATA_CENTER_REGION_NAME); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobVersionCleanup.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobVersionCleanup.java index b945dd5f484..7502b106dfa 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobVersionCleanup.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobVersionCleanup.java @@ -14,6 +14,7 @@ import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.server.VeniceServer; import com.linkedin.venice.utils.IntegrationTestPushUtils; @@ -22,7 +23,6 @@ import com.linkedin.venice.utils.Utils; import java.io.File; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -54,18 +54,20 @@ public void setUp() { controllerProps.put(ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED, false); controllerProps.put(PARTICIPANT_MESSAGE_STORE_ENABLED, true); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithEmergencySourceRegionSelection.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithEmergencySourceRegionSelection.java index a4113f88b79..a684df224fb 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithEmergencySourceRegionSelection.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithEmergencySourceRegionSelection.java @@ -21,6 +21,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.utils.IntegrationTestPushUtils; @@ -29,7 +30,6 @@ import com.linkedin.venice.utils.Utils; import java.io.File; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -85,18 +85,20 @@ public void setUp() { controllerProps.put(NATIVE_REPLICATION_SOURCE_FABRIC, "dc-0"); controllerProps.put(EMERGENCY_SOURCE_REGION, "dc-2"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java index 6a4fcc4d962..c7edff172a9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java @@ -68,6 +68,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; @@ -176,18 +177,20 @@ public void setUp() { controllerProps.put(PUSH_JOB_STATUS_STORE_CLUSTER_NAME, SYSTEM_STORE_CLUSTER); controllerProps.put(EMERGENCY_SOURCE_REGION, "dc-0"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); VeniceClusterWrapper clusterWrapper = diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java index 08f1862b5e6..c343eaabf27 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java @@ -19,6 +19,7 @@ import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.VeniceUserStoreType; @@ -67,18 +68,20 @@ public void setUp() { serverProperties.setProperty(SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE, "300"); Properties controllerProps = new Properties(); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllerRegionName = multiRegionMultiClusterWrapper.getParentRegionName() + ".parent"; clusterNames = multiRegionMultiClusterWrapper.getClusterNames(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRocksDBOffsetStore.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRocksDBOffsetStore.java index f6b9d61bdc2..6919506af51 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRocksDBOffsetStore.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRocksDBOffsetStore.java @@ -7,6 +7,7 @@ import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -26,7 +27,9 @@ public class TestRocksDBOffsetStore { @BeforeClass public void setUp() { - veniceCluster = ServiceFactory.getVeniceCluster(1, 0, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(1).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSeparateRealtimeTopicIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSeparateRealtimeTopicIngestion.java index 9656e6ebffb..7baed5aef98 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSeparateRealtimeTopicIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSeparateRealtimeTopicIngestion.java @@ -40,6 +40,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.ReadOnlySchemaRepository; @@ -63,7 +64,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -103,18 +103,20 @@ public void setUp() { serverProperties.put(ConfigKeys.SERVER_AA_WC_WORKLOAD_PARALLEL_PROCESSING_ENABLED, Boolean.toString(false)); Properties controllerProps = new Properties(); controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - REPLICATION_FACTOR, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSkipResultRecordTransformer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSkipResultRecordTransformer.java index 260d1d3646c..c4d4ce1bdac 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSkipResultRecordTransformer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSkipResultRecordTransformer.java @@ -1,6 +1,7 @@ package com.linkedin.venice.endToEnd; import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.venice.utils.lazy.Lazy; import java.io.IOException; @@ -18,8 +19,8 @@ public TestSkipResultRecordTransformer( Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci) { - super(storeVersion, keySchema, inputValueSchema, outputValueSchema, storeRecordsInDaVinci); + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); } @Override diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java index eab70f0733e..dd08d44cb2e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java @@ -14,6 +14,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreDataAudit; import com.linkedin.venice.meta.StoreInfo; @@ -26,7 +27,6 @@ import java.io.File; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -56,18 +56,18 @@ public class TestStaleDataVisibility { public void setUp() { Properties serverProperties = new Properties(); serverProperties.setProperty(ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1)); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childClusters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreBackupVersionDeletion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreBackupVersionDeletion.java index a988b7488f3..280d365b483 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreBackupVersionDeletion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreBackupVersionDeletion.java @@ -17,6 +17,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.Version; @@ -27,7 +28,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -60,17 +60,20 @@ public void setUp() { Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); StoreBackupVersionCleanupService.setWaitTimeDeleteRepushSourceVersion(10); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); veniceHelixAdmin = diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreGraveyardCleanupService.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreGraveyardCleanupService.java index 1e65af29e15..9eb2d1bcec4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreGraveyardCleanupService.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreGraveyardCleanupService.java @@ -13,12 +13,12 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreGraveyard; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -49,18 +49,19 @@ public void setUp() { parentControllerProperties.put(CONTROLLER_STORE_GRAVEYARD_CLEANUP_SLEEP_INTERVAL_BETWEEN_LIST_FETCH_MINUTES, 0); parentControllerProperties.put(CONTROLLER_STORE_GRAVEYARD_CLEANUP_DELAY_MINUTES, -1); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(parentControllerProperties) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java index 6c257103430..e52b9581f4c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java @@ -54,6 +54,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; @@ -133,19 +134,21 @@ public void setUp() throws Exception { // 1 parent controller, 1 child region, 2 clusters per child region, 2 servers per cluster // RF=2 to test both leader and follower SNs - twoLayerMultiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 2, - 1, - 1, - 2, - 1, - 2, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.of(serverProperties), - false, - true); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(2) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .sslToStorageNodes(true) + .forkServer(false) + .parentControllerProperties(parentControllerProperties) + .childControllerProperties(null) + .serverProperties(serverProperties); + twoLayerMultiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); multiClusterWrapper = twoLayerMultiRegionMultiClusterWrapper.getChildRegions().get(0); String[] clusterNames = multiClusterWrapper.getClusterNames(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStringRecordTransformer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStringRecordTransformer.java index 7c6fea86b3f..9b04152af3f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStringRecordTransformer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStringRecordTransformer.java @@ -1,6 +1,7 @@ package com.linkedin.venice.endToEnd; import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.venice.utils.lazy.Lazy; import java.io.IOException; @@ -12,20 +13,22 @@ public class TestStringRecordTransformer extends DaVinciRecordTransformer { private final Map inMemoryDB = new HashMap<>(); + private int transformInvocationCount = 0; public TestStringRecordTransformer( int storeVersion, Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci) { - super(storeVersion, keySchema, inputValueSchema, outputValueSchema, storeRecordsInDaVinci); + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); } @Override public DaVinciRecordTransformerResult transform(Lazy key, Lazy value) { String valueStr = convertUtf8ToString(value.get()); String transformedValue = valueStr + "Transformed"; + transformInvocationCount++; return new DaVinciRecordTransformerResult<>(DaVinciRecordTransformerResult.Result.TRANSFORMED, transformedValue); } @@ -62,6 +65,10 @@ public void put(Integer key, String value) { inMemoryDB.put(key, value); } + public int getTransformInvocationCount() { + return transformInvocationCount; + } + @Override public void close() throws IOException { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStuckConsumerRepair.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStuckConsumerRepair.java index 50d83ae91dc..2afa701ae5b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStuckConsumerRepair.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStuckConsumerRepair.java @@ -37,6 +37,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -79,7 +80,16 @@ public void setUp() { private static VeniceClusterWrapper setUpCluster() { Properties extraProperties = new Properties(); extraProperties.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 1, 1, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); // Add Venice Router Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSuperSetSchemaRegistration.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSuperSetSchemaRegistration.java index 7259546355c..1bb4b90e541 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSuperSetSchemaRegistration.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSuperSetSchemaRegistration.java @@ -9,6 +9,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; @@ -83,7 +84,9 @@ private static Schema writeComplicatedAvroFileWithUserSchema( @BeforeClass public void setUp() { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(); // Now with SSL! + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); // Now with SSL! } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestTopicWiseSharedConsumerPoolResilience.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestTopicWiseSharedConsumerPoolResilience.java index 9ad1201ca8a..bdea83ce042 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestTopicWiseSharedConsumerPoolResilience.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestTopicWiseSharedConsumerPoolResilience.java @@ -18,6 +18,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.meta.Version; @@ -54,7 +55,16 @@ public void setUp() { extraProperties.setProperty( SERVER_SHARED_CONSUMER_ASSIGNMENT_STRATEGY, KafkaConsumerService.ConsumerAssignmentStrategy.TOPIC_WISE_SHARED_CONSUMER_ASSIGNMENT_STRATEGY.name()); - veniceCluster = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnchangedResultRecordTransformer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnchangedResultRecordTransformer.java index a93a25a5ce9..7621a0663c8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnchangedResultRecordTransformer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnchangedResultRecordTransformer.java @@ -1,6 +1,7 @@ package com.linkedin.venice.endToEnd; import com.linkedin.davinci.client.DaVinciRecordTransformer; +import com.linkedin.davinci.client.DaVinciRecordTransformerConfig; import com.linkedin.davinci.client.DaVinciRecordTransformerResult; import com.linkedin.venice.utils.lazy.Lazy; import java.io.IOException; @@ -14,8 +15,8 @@ public TestUnchangedResultRecordTransformer( Schema keySchema, Schema inputValueSchema, Schema outputValueSchema, - boolean storeRecordsInDaVinci) { - super(storeVersion, keySchema, inputValueSchema, outputValueSchema, storeRecordsInDaVinci); + DaVinciRecordTransformerConfig recordTransformerConfig) { + super(storeVersion, keySchema, inputValueSchema, outputValueSchema, recordTransformerConfig); } @Override diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnusedValueSchemaCleanup.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnusedValueSchemaCleanup.java index 9d166ec95d5..c92cf664f4b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnusedValueSchemaCleanup.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnusedValueSchemaCleanup.java @@ -18,6 +18,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.utils.IntegrationTestPushUtils; @@ -27,7 +28,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -62,17 +62,20 @@ public void setUp() { Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); veniceHelixAdmin = diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVsonStoreBatch.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVsonStoreBatch.java index 16f97ddb984..e143bcef08f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVsonStoreBatch.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVsonStoreBatch.java @@ -25,6 +25,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.schema.vson.VsonAvroSchemaAdapter; @@ -69,7 +70,9 @@ public class TestVsonStoreBatch { @BeforeClass public void setUp() { - veniceCluster = ServiceFactory.getVeniceCluster(); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); controllerClient = new ControllerClient( veniceCluster.getClusterName(), veniceCluster.getLeaderVeniceController().getControllerUrl()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java index 60da54bae1f..2b0bbcbee17 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java @@ -11,6 +11,7 @@ import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; @@ -72,8 +73,15 @@ public void testFeatureFlagSingleDC() { @Test(timeOut = 90 * Time.MS_PER_SECOND) public void testFeatureFlagMultipleDC() { + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(0); try (VeniceTwoLayerMultiRegionMultiClusterWrapper twoLayerMultiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(1, 1, 1, 1, 1, 0)) { + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build())) { VeniceMultiClusterWrapper multiCluster = twoLayerMultiRegionMultiClusterWrapper.getChildRegions().get(0); VeniceControllerWrapper parentController = twoLayerMultiRegionMultiClusterWrapper.getParentControllers().get(0); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientIndividualFeatureConfigurationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientIndividualFeatureConfigurationTest.java index bed4dcb674f..4c35f686eb4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientIndividualFeatureConfigurationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/FastClientIndividualFeatureConfigurationTest.java @@ -451,7 +451,8 @@ public void testLongTailRetryManagerStats() throws IOException, ExecutionExcepti InstanceHealthMonitorConfig.builder() .setClient(r2Client) .setRoutingRequestDefaultTimeoutMS(10000) - .build())); + .build())) + .setRetryBudgetEnabled(true); String multiKeyLongTailRetryManagerStatsPrefix = ".multi-key-long-tail-retry-manager-" + storeName + "--"; String singleKeyLongTailRetryManagerStatsPrefix = ".single-key-long-tail-retry-manager-" + storeName + "--"; MetricsRepository clientMetric = new MetricsRepository(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/RequestBasedMetadataIntegrationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/RequestBasedMetadataIntegrationTest.java index 79a63e11380..ae20ae15af9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/RequestBasedMetadataIntegrationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/RequestBasedMetadataIntegrationTest.java @@ -15,6 +15,7 @@ import com.linkedin.venice.fastclient.utils.ClientTestUtils; import com.linkedin.venice.integration.utils.D2TestUtils; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.OnlineInstanceFinder; @@ -61,7 +62,16 @@ public void setUp() throws Exception { Utils.thisIsLocalhost(); Properties props = new Properties(); props.put(SERVER_HTTP2_INBOUND_ENABLED, "true"); - veniceCluster = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 100, true, false, props); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .extraProperties(props) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); r2Client = ClientTestUtils.getR2Client(); d2Client = D2TestUtils.getAndStartHttpsD2Client(veniceCluster.getZk().getAddress()); storeName = veniceCluster.createStore(KEY_COUNT); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestControllerKMERegistrationFromMessageHeader.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestControllerKMERegistrationFromMessageHeader.java index c7c0388affd..817e7af0439 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestControllerKMERegistrationFromMessageHeader.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestControllerKMERegistrationFromMessageHeader.java @@ -8,13 +8,13 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; import com.linkedin.venice.serialization.avro.KafkaValueSerializer; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -45,17 +45,19 @@ public void setUp() { controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 3); controllerProps.put(DEFAULT_PARTITION_SIZE, 1024); controllerProps.put(KME_REGISTRATION_FROM_MESSAGE_HEADER_ENABLED, true); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 3, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.empty()); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(3) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestServerKMERegistrationFromMessageHeader.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestServerKMERegistrationFromMessageHeader.java index 7009d74ef8e..5384fa538c1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestServerKMERegistrationFromMessageHeader.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestServerKMERegistrationFromMessageHeader.java @@ -6,6 +6,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -52,8 +53,15 @@ public class TestServerKMERegistrationFromMessageHeader { @BeforeClass public void setUp() { - cluster = - ServiceFactory.getVeniceCluster(numOfController, 0, numOfRouters, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(numOfController) + .numberOfServers(0) + .numberOfRouters(numOfRouters) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); clusterName = cluster.getClusterName(); Properties serverProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/LeaderFollowerThreadPoolTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/LeaderFollowerThreadPoolTest.java index 50b23b5d4d8..8dd759068a4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/LeaderFollowerThreadPoolTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/LeaderFollowerThreadPoolTest.java @@ -5,6 +5,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.pushmonitor.ExecutionStatus; @@ -45,8 +46,15 @@ public void setUp() { int numOfController = 1; int numOfServers = 0; int numOfRouters = 1; - cluster = ServiceFactory - .getVeniceCluster(numOfController, numOfServers, numOfRouters, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(numOfController) + .numberOfServers(numOfServers) + .numberOfRouters(numOfRouters) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); } @AfterMethod diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/TestRebalanceByDefaultStrategy.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/TestRebalanceByDefaultStrategy.java index 8bb5bca6187..7100ab88514 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/TestRebalanceByDefaultStrategy.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/TestRebalanceByDefaultStrategy.java @@ -6,6 +6,7 @@ import com.linkedin.venice.helix.HelixState; import com.linkedin.venice.helix.Replica; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.pushmonitor.ExecutionStatus; import com.linkedin.venice.utils.TestUtils; @@ -47,14 +48,16 @@ public class TestRebalanceByDefaultStrategy { @BeforeClass public void setUp() { - cluster = ServiceFactory.getVeniceCluster( - numberOfController, - numberOfServer, - numberOfRouter, - replicationFactor, - partitionSize, - false, - false); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(numberOfServer) + .numberOfRouters(numberOfRouter) + .replicationFactor(replicationFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); String storeName = Utils.getUniqueString("testRollingUpgrade"); long storageQuota = (long) partitionSize * partitionNumber; cluster.getNewStore(storeName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java index bc7cee1128b..2e10e3dea20 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java @@ -31,6 +31,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.kafka.protocol.KafkaMessageEnvelope; import com.linkedin.venice.message.KafkaKey; @@ -58,7 +59,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -90,18 +90,20 @@ public void setUp() { Properties serverProperties = new Properties(); Properties controllerProps = new Properties(); controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 4, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(4) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/StorageNodeServiceTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/StorageNodeServiceTest.java index 0e7295e9987..ebc0a2d9188 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/StorageNodeServiceTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/StorageNodeServiceTest.java @@ -2,6 +2,7 @@ import com.linkedin.venice.httpclient.HttpClientUtils; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.utils.SslUtils; @@ -21,11 +22,18 @@ public class StorageNodeServiceTest { @Test public void storageServerRespondsToRequests() throws ExecutionException, InterruptedException, IOException { Utils.thisIsLocalhost(); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(1) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); try ( CloseableHttpAsyncClient client = HttpClientUtils.getMinimalHttpClient(1, 1, Optional.of(SslUtils.getVeniceLocalSslFactory())); - VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(1, 1, 0, 1, 100, true, false)) { - + VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options)) { client.start(); VeniceServerWrapper sslServer = venice.getVeniceServers().get(0); // This should work, talking ssl to an ssl storage node diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java index 6a51152ef6e..fa7b59b7dc5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java @@ -281,7 +281,7 @@ public static MockHttpServerWrapper getMockHttpServer(String serviceName) { /** * Start up a testing Venice cluster in another process. * - * The reason to call this method instead of other {@link #getVeniceCluster()} methods is + * The reason to call this method instead of other {@link #getVeniceCluster(VeniceClusterCreateOptions)} methods is * when one wants to maximize its testing environment isolation. * Example usage: {@literal com.linkedin.venice.benchmark.IngestionBenchmarkWithTwoProcesses} * @@ -294,7 +294,7 @@ public static void startVeniceClusterInAnotherProcess(String clusterInfoFilePath /** * Start up a testing Venice cluster in another process. * - * The reason to call this method instead of other {@link #getVeniceCluster()} methods is + * The reason to call this method instead of other {@link #getVeniceCluster(VeniceClusterCreateOptions)} methods is * when one wants to maximize its testing environment isolation. * Example usage: {@literal com.linkedin.venice.benchmark.IngestionBenchmarkWithTwoProcesses} * @@ -319,193 +319,10 @@ public static VeniceClusterWrapper getVeniceCluster(VeniceClusterCreateOptions o return getService(VeniceClusterWrapper.SERVICE_NAME, VeniceClusterWrapper.generateService(options)); } - /** - * @deprecated - *

Use {@link ServiceFactory#getVeniceCluster(VeniceClusterCreateOptions)} instead. - */ - @Deprecated - public static VeniceClusterWrapper getVeniceCluster() { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); - return getVeniceCluster(options); - } - - @Deprecated - public static VeniceClusterWrapper getVeniceCluster( - int numberOfControllers, - int numberOfServers, - int numberOfRouters) { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .build(); - return getVeniceCluster(options); - } - - @Deprecated - public static VeniceClusterWrapper getVeniceCluster( - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor) { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .build(); - return getVeniceCluster(options); - } - - @Deprecated - public static VeniceClusterWrapper getVeniceCluster( - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - int partitionSize, - boolean sslToStorageNodes, - boolean sslToKafka, - Properties extraProperties) { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .partitionSize(partitionSize) - .sslToStorageNodes(sslToStorageNodes) - .sslToKafka(sslToKafka) - .extraProperties(extraProperties) - .build(); - return getVeniceCluster(options); - } - - @Deprecated - public static VeniceClusterWrapper getVeniceCluster( - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - int partitionSize, - boolean sslToStorageNodes, - boolean sslToKafka) { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .partitionSize(partitionSize) - .sslToStorageNodes(sslToStorageNodes) - .sslToKafka(sslToKafka) - .build(); - return getVeniceCluster(options); - } - public static VeniceMultiClusterWrapper getVeniceMultiClusterWrapper(VeniceMultiClusterCreateOptions options) { return getService(VeniceMultiClusterWrapper.SERVICE_NAME, VeniceMultiClusterWrapper.generateService(options)); } - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters) { - VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) - .numberOfClusters(numberOfClustersInEachRegion) - .numberOfParentControllers(numberOfParentControllers) - .numberOfChildControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters); - return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); - } - - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - Optional parentControllerProps, - Optional childControllerProperties, - Optional serverProps) { - return getVeniceTwoLayerMultiRegionMultiClusterWrapper( - numberOfRegions, - numberOfClustersInEachRegion, - numberOfParentControllers, - numberOfControllers, - numberOfServers, - numberOfRouters, - replicationFactor, - parentControllerProps, - childControllerProperties, - serverProps, - false); - } - - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - Optional parentControllerProps, - Optional childControllerProperties, - Optional serverProps, - boolean forkServer, - boolean sslToStorageNodes) { - VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) - .numberOfClusters(numberOfClustersInEachRegion) - .numberOfParentControllers(numberOfParentControllers) - .numberOfChildControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .sslToStorageNodes(sslToStorageNodes) - .forkServer(forkServer); - - parentControllerProps.ifPresent(optionsBuilder::parentControllerProperties); - childControllerProperties.ifPresent(optionsBuilder::childControllerProperties); - serverProps.ifPresent(optionsBuilder::serverProperties); - return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); - } - - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - Optional parentControllerProps, - Optional childControllerProperties, - Optional serverProps, - boolean forkServer) { - VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) - .numberOfClusters(numberOfClustersInEachRegion) - .numberOfParentControllers(numberOfParentControllers) - .numberOfChildControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .forkServer(forkServer); - - parentControllerProps.ifPresent(optionsBuilder::parentControllerProperties); - childControllerProperties.ifPresent(optionsBuilder::childControllerProperties); - serverProps.ifPresent(optionsBuilder::serverProperties); - return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); - } - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( VeniceMultiRegionClusterCreateOptions options) { return getService( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/TestFileDescriptorLeak.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/TestFileDescriptorLeak.java index c32c4419e38..9e70e59d808 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/TestFileDescriptorLeak.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/TestFileDescriptorLeak.java @@ -37,7 +37,9 @@ public void testKafkaBrokerLeak() { @Test(invocationCount = 20, groups = { "flaky" }) public void testVeniceClusterLeak() { - try (VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster()) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + try (VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options)) { LOGGER.info("Created VeniceClusterWrapper: {}", veniceClusterWrapper.getClusterName()); } } @@ -46,7 +48,9 @@ public void testVeniceClusterLeak() { @BeforeClass public void setUpReusableCluster() { - this.veniceClusterWrapper = ServiceFactory.getVeniceCluster(1, 2, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(2).numberOfRouters(1).build(); + this.veniceClusterWrapper = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java index e1a28520f71..198ae0bf0bf 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java @@ -589,6 +589,7 @@ public VeniceControllerWrapper addVeniceController(Properties properties) { .clusterToD2(clusterToD2) .clusterToServerD2(clusterToServerD2) .extraProperties(properties) + .dynamicAccessController(options.getAccessController()) .build()); synchronized (this) { veniceControllerWrappers.put(veniceControllerWrapper.getPort(), veniceControllerWrapper); @@ -1172,8 +1173,16 @@ public static void main(String[] args) throws IOException { Utils.thisIsLocalhost(); Properties extraProperties = new Properties(); extraProperties.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, numberOfPartitions); - VeniceClusterWrapper veniceClusterWrapper = - ServiceFactory.getVeniceCluster(1, 1, 1, 1, 10 * 1024 * 1024, false, false, extraProperties); + VeniceClusterCreateOptions options1 = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(10 * 1024 * 1024) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options1); String storeName = Utils.getUniqueString("storeForMainMethodOf" + VeniceClusterWrapper.class.getSimpleName()); String controllerUrl = veniceClusterWrapper.getRandomVeniceController().getControllerUrl(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java index d02a19a1b45..e32457bb6eb 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java @@ -10,6 +10,7 @@ import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_STORAGE_NODES; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.STANDALONE_REGION_NAME; +import com.linkedin.venice.acl.DynamicAccessController; import java.util.Collections; import java.util.Map; import java.util.Properties; @@ -37,6 +38,7 @@ public class VeniceMultiClusterCreateOptions { private final PubSubBrokerWrapper pubSubBrokerWrapper; private final Properties childControllerProperties; private final Properties extraProperties; + private final DynamicAccessController accessController; public String getRegionName() { return regionName; @@ -122,6 +124,10 @@ public Properties getExtraProperties() { return extraProperties; } + public DynamicAccessController getAccessController() { + return accessController; + } + @Override public String toString() { return new StringBuilder().append("VeniceMultiClusterCreateOptions - ") @@ -212,6 +218,7 @@ private VeniceMultiClusterCreateOptions(Builder builder) { extraProperties = builder.extraProperties; forkServer = builder.forkServer; kafkaClusterMap = builder.kafkaClusterMap; + accessController = builder.accessController; } public static class Builder { @@ -236,6 +243,7 @@ public static class Builder { private PubSubBrokerWrapper pubSubBrokerWrapper; private Properties childControllerProperties; private Properties extraProperties; + private DynamicAccessController accessController; public Builder numberOfClusters(int numberOfClusters) { this.numberOfClusters = numberOfClusters; @@ -346,6 +354,11 @@ public Builder extraProperties(Properties extraProperties) { return this; } + public Builder accessController(DynamicAccessController accessController) { + this.accessController = accessController; + return this; + } + private void addDefaults() { if (numberOfClusters == 0) { numberOfClusters = 1; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java index 88baea83932..21e325b1d92 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java @@ -117,6 +117,7 @@ static ServiceProvider generateService(VeniceMultiClu .clusterToServerD2(clusterToServerD2) .sslToKafka(false) .d2Enabled(true) + .dynamicAccessController(options.getAccessController()) .extraProperties(controllerProperties) .build(); for (int i = 0; i < options.getNumberOfControllers(); i++) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java index c2dafc85268..08b080d654a 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java @@ -7,6 +7,7 @@ import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_KAFKA; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_STORAGE_NODES; +import com.linkedin.venice.acl.DynamicAccessController; import com.linkedin.venice.authorization.AuthorizerService; import java.util.Properties; @@ -28,6 +29,7 @@ public class VeniceMultiRegionClusterCreateOptions { private final AuthorizerService parentAuthorizerService; private final String parentVeniceZkBasePath; private final String childVeniceZkBasePath; + private final DynamicAccessController accessController; public int getNumberOfRegions() { return numberOfRegions; @@ -93,6 +95,10 @@ public String getChildVeniceZkBasePath() { return childVeniceZkBasePath; } + public DynamicAccessController getAccessController() { + return accessController; + } + @Override public String toString() { return new StringBuilder().append("VeniceMultiClusterCreateOptions - ") @@ -163,6 +169,7 @@ private VeniceMultiRegionClusterCreateOptions(Builder builder) { parentAuthorizerService = builder.parentAuthorizerService; parentVeniceZkBasePath = builder.parentVeniceZkBasePath; childVeniceZkBasePath = builder.childVeniceZkBasePath; + accessController = builder.accessController; } public static class Builder { @@ -182,6 +189,7 @@ public static class Builder { private AuthorizerService parentAuthorizerService; private String parentVeniceZkBasePath = "/"; private String childVeniceZkBasePath = "/"; + private DynamicAccessController accessController; public Builder numberOfRegions(int numberOfRegions) { this.numberOfRegions = numberOfRegions; @@ -271,6 +279,11 @@ public Builder childVeniceZkBasePath(String veniceZkBasePath) { return this; } + public Builder accessController(DynamicAccessController accessController) { + this.accessController = accessController; + return this; + } + private void addDefaults() { if (numberOfRegions == 0) { numberOfRegions = 1; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java index 8a36bb03f67..dec15c6ad83 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceServerWrapper.java @@ -51,6 +51,7 @@ import com.linkedin.venice.server.VeniceServerContext; import com.linkedin.venice.servicediscovery.ServiceDiscoveryAnnouncer; import com.linkedin.venice.tehuti.MetricsAware; +import com.linkedin.venice.tehuti.MockTehutiReporter; import com.linkedin.venice.utils.ForkedJavaProcess; import com.linkedin.venice.utils.PropertyBuilder; import com.linkedin.venice.utils.SslUtils; @@ -58,6 +59,7 @@ import com.linkedin.venice.utils.Utils; import com.linkedin.venice.utils.VeniceProperties; import com.linkedin.venice.utils.metrics.MetricsRepositoryUtils; +import io.tehuti.Metric; import io.tehuti.metrics.MetricsRepository; import java.io.File; import java.io.IOException; @@ -78,6 +80,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.testng.Assert; /** @@ -454,12 +457,28 @@ protected void internalStart() throws Exception { @Override protected void internalStop() throws Exception { if (!forkServer) { + verifyHelixParticipantServicePoolMetricsReporting(veniceServer); veniceServer.shutdown(); } else { serverProcess.destroy(); } } + private void verifyHelixParticipantServicePoolMetricsReporting(VeniceServer veniceServer) { + MetricsRepository metricsRepository = veniceServer.getMetricsRepository(); + MockTehutiReporter reporter = new MockTehutiReporter(); + metricsRepository.addReporter(reporter); + Metric activeThreadNumber = reporter.query(".Venice_L/F_ST_thread_pool--active_thread_number.LambdaStat"); + Assert.assertNotNull(activeThreadNumber); + Assert.assertTrue(activeThreadNumber.value() >= 0); + Metric maxThreadNumber = reporter.query(".Venice_L/F_ST_thread_pool--max_thread_number.LambdaStat"); + Assert.assertNotNull(maxThreadNumber); + Assert.assertTrue(maxThreadNumber.value() > 0); + Metric queuedTaskNumber = reporter.query(".Venice_L/F_ST_thread_pool--queued_task_number.LambdaStat"); + Assert.assertNotNull(queuedTaskNumber); + Assert.assertTrue(queuedTaskNumber.value() >= 0); + } + @Override protected void newProcess() throws Exception { if (forkServer) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java index 547c3db34a2..4d09daf5861 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java @@ -224,6 +224,7 @@ static ServiceProvider generateSer .clusterToServerD2(clusterToServerD2) .regionName(parentRegionName) .authorizerService(options.getParentAuthorizerService()) + .dynamicAccessController(options.getAccessController()) .build(); // Create parentControllers for multi-cluster for (int i = 0; i < options.getNumberOfParentControllers(); i++) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartController.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartController.java index 286de4b2c01..179f5bcadae 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartController.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartController.java @@ -5,6 +5,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.pushmonitor.ExecutionStatus; @@ -31,7 +32,12 @@ public void setUp() { int numberOfServer = 1; int numberOfRouter = 1; - cluster = ServiceFactory.getVeniceCluster(numberOfController, numberOfServer, numberOfRouter); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(numberOfServer) + .numberOfRouters(numberOfRouter) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); } @AfterMethod diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartRouter.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartRouter.java index bc48bffbccd..495c28e9110 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartRouter.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartRouter.java @@ -5,6 +5,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.RoutingDataRepository; @@ -30,7 +31,12 @@ public void setUp() { int numberOfServer = 1; int numberOfRouter = 2; - cluster = ServiceFactory.getVeniceCluster(numberOfController, numberOfServer, numberOfRouter); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(numberOfServer) + .numberOfRouters(numberOfRouter) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java index f0b0b4a133a..54eb22f5974 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java @@ -35,6 +35,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; @@ -123,18 +124,18 @@ public void setUp() throws Exception { serverProperties.put( CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_COLOS, - 1, - 1, - 2, - numServers, - 1, - NUMBER_OF_REPLICAS, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_COLOS) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(2) + .numberOfServers(numServers) + .numberOfRouters(1) + .replicationFactor(NUMBER_OF_REPLICAS) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); List childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java index 60d2eb5f31a..301c508e4c4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java @@ -14,6 +14,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -74,8 +75,16 @@ public void setUp() { int numberOfRouter = 1; int replicaFactor = 1; int partitionSize = 1000; - cluster = ServiceFactory - .getVeniceCluster(numberOfController, 0, numberOfRouter, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(0) + .numberOfRouters(numberOfRouter) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); pubSubProducerAdapterFactory = cluster.getPubSubBrokerWrapper().getPubSubClientsFactory().getProducerAdapterFactory(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/ReplicaFailoverTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/ReplicaFailoverTest.java index df162487768..6763cc66f93 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/ReplicaFailoverTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/ReplicaFailoverTest.java @@ -5,6 +5,7 @@ import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -52,7 +53,12 @@ public class ReplicaFailoverTest { @BeforeClass public void setUp() throws Exception { Utils.thisIsLocalhost(); - cluster = ServiceFactory.getVeniceCluster(1, REPLICATION_FACTOR, 0, REPLICATION_FACTOR); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(REPLICATION_FACTOR) + .numberOfRouters(0) + .replicationFactor(REPLICATION_FACTOR) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); Properties props = new Properties(); props.setProperty(ConfigKeys.ROUTER_STATEFUL_HEALTHCHECK_ENABLED, "true"); props.setProperty(ConfigKeys.ROUTER_ASYNC_START_ENABLED, "true"); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java index aaa1da09a29..5ad00428c56 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java @@ -7,7 +7,6 @@ import static com.linkedin.venice.ConfigKeys.OFFLINE_JOB_START_TIMEOUT_MS; import static com.linkedin.venice.ConfigKeys.PERSISTENCE_TYPE; import static com.linkedin.venice.ConfigKeys.PUSH_STATUS_STORE_ENABLED; -import static com.linkedin.venice.client.store.ClientFactory.getTransportClient; import static com.linkedin.venice.utils.TestUtils.assertCommand; import com.linkedin.d2.balancer.D2Client; @@ -18,7 +17,6 @@ import com.linkedin.venice.blobtransfer.BlobFinder; import com.linkedin.venice.blobtransfer.BlobPeersDiscoveryResponse; import com.linkedin.venice.blobtransfer.DaVinciBlobFinder; -import com.linkedin.venice.client.store.AvroGenericStoreClientImpl; import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.common.VeniceSystemStoreType; import com.linkedin.venice.controllerapi.ControllerClient; @@ -31,6 +29,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -46,7 +45,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -65,7 +63,9 @@ public class TestBlobDiscovery { private static final String INT_KEY_SCHEMA = "\"int\""; private static final String INT_VALUE_SCHEMA = "\"int\""; String clusterName; + String clusterName2; String storeName; + String storeName2; private VeniceMultiClusterWrapper multiClusterVenice; private VeniceTwoLayerMultiRegionMultiClusterWrapper multiRegionMultiClusterWrapper; D2Client daVinciD2; @@ -81,18 +81,18 @@ public void setUp() { Properties props = new Properties(); props.put(OFFLINE_JOB_START_TIMEOUT_MS, "180000"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 2, - 1, - 1, - 3, - 1, - 3, - Optional.of(props), - Optional.empty(), - Optional.empty(), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(2) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(3) + .numberOfRouters(1) + .replicationFactor(3) + .forkServer(false) + .parentControllerProperties(props); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); multiClusterVenice = multiRegionMultiClusterWrapper.getChildRegions().get(0); String[] clusterNames = multiClusterVenice.getClusterNames(); @@ -102,7 +102,15 @@ public void setUp() { .collect(Collectors.joining(",")); clusterName = clusterNames[0]; + clusterName2 = clusterNames[1]; storeName = Utils.getUniqueString("test-store"); + storeName2 = Utils.getUniqueString("test-store"); + LOGGER.info( + "Cluster 1 is {}, Cluster 2 is {}, Store 1 is {}, Store 2 is {}", + clusterName, + clusterName2, + storeName, + storeName2); List pubSubBrokerWrappers = multiClusterVenice.getClusters() .values() @@ -112,6 +120,7 @@ public void setUp() { Map additionalPubSubProperties = PubSubBrokerWrapper.getBrokerDetailsForClients(pubSubBrokerWrappers); + // create store 1 in cluster 1 try (ControllerClient parentControllerClient = new ControllerClient(clusterName, parentControllerURLs)) { assertCommand(parentControllerClient.createNewStore(storeName, "venice-test", INT_KEY_SCHEMA, INT_VALUE_SCHEMA)); @@ -141,6 +150,36 @@ public void setUp() { multiClusterVenice.getClusters().get(clusterName).refreshAllRouterMetaData(); } + // create store 2 in cluster 2 + try (ControllerClient parentControllerClient = new ControllerClient(clusterName2, parentControllerURLs)) { + assertCommand(parentControllerClient.createNewStore(storeName2, "venice-test", INT_KEY_SCHEMA, INT_VALUE_SCHEMA)); + + PubSubProducerAdapterFactory pubSubProducerAdapterFactory = multiClusterVenice.getClusters() + .get(clusterName2) + .getPubSubBrokerWrapper() + .getPubSubClientsFactory() + .getProducerAdapterFactory(); + + VersionCreationResponse response = TestUtils.createVersionWithBatchData( + parentControllerClient, + storeName2, + INT_KEY_SCHEMA, + INT_VALUE_SCHEMA, + IntStream.range(0, 10).mapToObj(i -> new AbstractMap.SimpleEntry<>(i, 0)), + pubSubProducerAdapterFactory, + additionalPubSubProperties); + + // Verify the data can be ingested by classical Venice before proceeding. + TestUtils.waitForNonDeterministicPushCompletion( + response.getKafkaTopic(), + parentControllerClient, + 30, + TimeUnit.SECONDS); + + makeSureSystemStoresAreOnline(parentControllerClient, storeName2); + multiClusterVenice.getClusters().get(clusterName2).refreshAllRouterMetaData(); + } + VeniceProperties backendConfig = new PropertyBuilder().put(DATA_BASE_PATH, Utils.getTempDataDirectory().getAbsolutePath()) .put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB) @@ -153,17 +192,24 @@ public void setUp() { DaVinciConfig daVinciConfig = new DaVinciConfig(); daVinciD2 = D2TestUtils.getAndStartD2Client(multiClusterVenice.getZkServerWrapper().getAddress()); + // create one DVC client which subscribes to all partitions for store 1 and store 2 try (CachingDaVinciClientFactory factory = new CachingDaVinciClientFactory( daVinciD2, VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME, new MetricsRepository(), backendConfig)) { + // subscribe to all partitions for store 1 List> clients = new ArrayList<>(); DaVinciClient client = factory.getAndStartGenericAvroClient(storeName, daVinciConfig); client.subscribeAll().get(); clients.add(client); + // subscribe to all partitions for store 2 + DaVinciClient client2 = factory.getAndStartGenericAvroClient(storeName2, daVinciConfig); + client2.subscribeAll().get(); + clients.add(client2); // This is a very dumb and basic assertion that's only purpose is to get static analysis to not be mad Assert.assertTrue(clients.get(0).getPartitionCount() > 0); + Assert.assertTrue(clients.get(1).getPartitionCount() > 0); } catch (ExecutionException | InterruptedException e) { throw new VeniceException(e); } @@ -181,17 +227,36 @@ public void testBlobDiscovery() throws Exception { TestUtils.waitForNonDeterministicAssertion(1, TimeUnit.MINUTES, true, () -> { veniceClusterWrapper.updateStore(storeName, new UpdateStoreQueryParams().setBlobTransferEnabled(true)); }); + VeniceClusterWrapper veniceClusterWrapper2 = multiClusterVenice.getClusters().get(clusterName2); + TestUtils.waitForNonDeterministicAssertion(1, TimeUnit.MINUTES, true, () -> { + veniceClusterWrapper2.updateStore(storeName2, new UpdateStoreQueryParams().setBlobTransferEnabled(true)); + }); + // Even if the config here only use "storeName", + // the DaVinciBlobFinder should be able to discover both stores ClientConfig clientConfig = new ClientConfig(storeName).setD2Client(daVinciD2) .setD2ServiceName(VeniceRouterWrapper.CLUSTER_DISCOVERY_D2_SERVICE_NAME) .setMetricsRepository(new MetricsRepository()); - BlobFinder daVinciBlobFinder = - new DaVinciBlobFinder(new AvroGenericStoreClientImpl<>(getTransportClient(clientConfig), false, clientConfig)); + // Checking if the DVC created at the beginning is found by the DaVinciBlobFinder + // when try to find store1's peers. + BlobFinder daVinciBlobFinder = new DaVinciBlobFinder(clientConfig); TestUtils.waitForNonDeterministicAssertion(1, TimeUnit.MINUTES, true, () -> { BlobPeersDiscoveryResponse response = daVinciBlobFinder.discoverBlobPeers(storeName, 1, 1); Assert.assertNotNull(response); List hostNames = response.getDiscoveryResult(); + LOGGER.info("Discovered hosts: {} for store1 {}", hostNames, storeName); + Assert.assertNotNull(hostNames); + Assert.assertEquals(hostNames.size(), 1); + }); + + // Checking if the DVC created at the beginning is found by the DaVinciBlobFinder + // when try to find store2's peers. + TestUtils.waitForNonDeterministicAssertion(1, TimeUnit.MINUTES, true, () -> { + BlobPeersDiscoveryResponse response = daVinciBlobFinder.discoverBlobPeers(storeName2, 1, 1); + Assert.assertNotNull(response); + List hostNames = response.getDiscoveryResult(); + LOGGER.info("Discovered hosts: {} for store2 {}", hostNames, storeName2); Assert.assertNotNull(hostNames); Assert.assertEquals(hostNames.size(), 1); }); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestConnectionWarmingForApacheAsyncClient.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestConnectionWarmingForApacheAsyncClient.java index 67ed54852be..aaf8e446be3 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestConnectionWarmingForApacheAsyncClient.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestConnectionWarmingForApacheAsyncClient.java @@ -3,6 +3,7 @@ import com.linkedin.venice.ConfigKeys; import com.linkedin.venice.client.exceptions.VeniceClientException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -26,7 +27,15 @@ public class TestConnectionWarmingForApacheAsyncClient { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 2, 0, 2, 100, true, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); Properties routerProperties = new Properties(); routerProperties.put(ConfigKeys.ROUTER_STORAGE_NODE_CLIENT_TYPE, StorageNodeClientType.APACHE_HTTP_ASYNC_CLIENT); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRead.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRead.java index 79968b4f759..c080531a6c8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRead.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRead.java @@ -28,6 +28,7 @@ import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.D2TestUtils; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -158,7 +159,16 @@ public void setUp() throws VeniceClientException, ExecutionException, Interrupte extraProperties.put(ConfigKeys.ROUTER_PER_STORE_ROUTER_QUOTA_BUFFER, 0.0); extraProperties.put(ConfigKeys.PARTICIPANT_MESSAGE_STORE_ENABLED, false); - veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 1, 2, 100, true, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); routerAddr = veniceCluster.getRandomRouterSslURL(); Properties serverProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRouterAsyncStart.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRouterAsyncStart.java index 80d3bf9273b..929b030c501 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRouterAsyncStart.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRouterAsyncStart.java @@ -3,6 +3,7 @@ import com.linkedin.venice.ConfigKeys; import com.linkedin.venice.client.exceptions.VeniceClientException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.router.httpclient.StorageNodeClientType; @@ -23,7 +24,9 @@ public class TestRouterAsyncStart { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestStreaming.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestStreaming.java index c793a65afc7..1d7bc992e9f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestStreaming.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestStreaming.java @@ -29,6 +29,7 @@ import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.D2TestUtils; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -111,7 +112,15 @@ public void setUp() throws InterruptedException, ExecutionException, VeniceClien System.setProperty("io.netty.leakDetection.level", "paranoid"); Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 2, 0, 2, 100, true, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); Properties serverFeatureProperties = new Properties(); serverFeatureProperties.put(VeniceServerWrapper.SERVER_ENABLE_SSL, "true"); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/samza/VeniceSystemFactoryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/samza/VeniceSystemFactoryTest.java index c5884bd25d8..af97bb1bf43 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/samza/VeniceSystemFactoryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/samza/VeniceSystemFactoryTest.java @@ -11,6 +11,7 @@ import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.schema.writecompute.WriteComputeSchemaConverter; @@ -43,7 +44,9 @@ public class VeniceSystemFactoryTest { @BeforeClass public void setUp() { - cluster = ServiceFactory.getVeniceCluster(); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/server/VeniceServerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/server/VeniceServerTest.java index 6fb1c6a615c..bf7dca1aa8f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/server/VeniceServerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/server/VeniceServerTest.java @@ -79,7 +79,9 @@ public class VeniceServerTest { @Test public void testStartServerWithDefaultConfigForTests() throws NoSuchFieldException, IllegalAccessException { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { TestVeniceServer server = cluster.getVeniceServers().get(0).getVeniceServer(); Assert.assertTrue(server.isStarted()); @@ -91,7 +93,9 @@ public void testStartServerWithDefaultConfigForTests() throws NoSuchFieldExcepti @Test public void testStartServerWhenEnableAllowlistCheckingFailed() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { Assert.assertThrows(VeniceException.class, () -> { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(true)); @@ -104,7 +108,9 @@ public void testStartServerWhenEnableAllowlistCheckingFailed() { @Test public void testStartServerWhenEnableAllowlistCheckingSuccessful() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(true)); featureProperties.setProperty(SERVER_IS_AUTO_JOIN, Boolean.toString(true)); @@ -115,7 +121,9 @@ public void testStartServerWhenEnableAllowlistCheckingSuccessful() { @Test public void testCheckBeforeJoinCluster() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { VeniceServerWrapper server = cluster.getVeniceServers().get(0); StorageEngineRepository repository = server.getVeniceServer().getStorageService().getStorageEngineRepository(); @@ -176,7 +184,9 @@ public void testCheckBeforeJoinCluster() { @Test public void testCheckBeforeJointClusterBeforeHelixInitializingCluster() throws Exception { Thread serverAddingThread = null; - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(0, 0, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(0).numberOfServers(0).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { serverAddingThread = new Thread(() -> { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(false)); @@ -200,7 +210,9 @@ public void testCheckBeforeJointClusterBeforeHelixInitializingCluster() throws E @Test public void testStartServerAndShutdownWithPartitionAssignmentVerification() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(true)); featureProperties.setProperty(SERVER_IS_AUTO_JOIN, Boolean.toString(true)); @@ -244,7 +256,12 @@ public void testMetadataFetchRequest() throws ExecutionException, InterruptedExc int servers = 6; int replicationFactor = 2; - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, servers, 0, replicationFactor); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(servers) + .numberOfRouters(0) + .replicationFactor(replicationFactor) + .build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); CloseableHttpAsyncClient client = HttpClientUtils.getMinimalHttpClient(1, 1, Optional.of(SslUtils.getVeniceLocalSslFactory()))) { @@ -351,7 +368,9 @@ public void testMetadataFetchRequest() throws ExecutionException, InterruptedExc @Test(dataProvider = "True-and-False", dataProviderClass = DataProviderUtils.class) public void testVeniceServerWithD2(boolean https) throws Exception { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { String storeName = cluster.createStore(1); Map clusterToServerD2 = cluster.getClusterToServerD2(); String d2ServiceName = clusterToServerD2.get(cluster.getClusterName()); @@ -397,7 +416,9 @@ public void testStartServerWithSystemSchemaInitialization() { @Test(timeOut = TOTAL_TIMEOUT_FOR_LONG_TEST_MS) public void testDropStorePartitionAsynchronously() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { VeniceServerWrapper server = cluster.getVeniceServers().get(0); Assert.assertTrue(server.getVeniceServer().isStarted()); @@ -435,7 +456,9 @@ public void testDropStorePartitionAsynchronously() { @Test(timeOut = TOTAL_TIMEOUT_FOR_LONG_TEST_MS) public void testDropStorePartitionSynchronously() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(true)); featureProperties.setProperty(SERVER_IS_AUTO_JOIN, Boolean.toString(true)); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/ReadComputeValidationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/ReadComputeValidationTest.java index 58cb7aa09fb..a64044f3428 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/ReadComputeValidationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/ReadComputeValidationTest.java @@ -18,6 +18,7 @@ import com.linkedin.venice.exceptions.ErrorType; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -94,7 +95,15 @@ public class ReadComputeValidationTest { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { - veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0, 2, 100, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); // Add one more server with fast-avro enabled Properties serverProperties = new Properties(); serverProperties.put(ConfigKeys.SERVER_COMPUTE_FAST_AVRO_ENABLED, true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/StorageNodeComputeTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/StorageNodeComputeTest.java index bf48f492a7c..ba498876267 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/StorageNodeComputeTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/StorageNodeComputeTest.java @@ -20,6 +20,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -97,7 +98,15 @@ enum AvroImpl { @BeforeClass(alwaysRun = true) public void setUp() throws InterruptedException, ExecutionException, VeniceClientException { - veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0, 2, 100, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); // Add one more server with all the bells and whistles: fast-avro, parallel batch get Properties serverProperties = new Properties(); serverProperties.put(ConfigKeys.SERVER_COMPUTE_FAST_AVRO_ENABLED, true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/TestEarlyTermination.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/TestEarlyTermination.java index e36062a8477..5b3665f94b6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/TestEarlyTermination.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/TestEarlyTermination.java @@ -12,6 +12,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.Version; @@ -50,7 +51,15 @@ public class TestEarlyTermination { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 0, 1, 2, 100, true, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); // Create store first storeName = Utils.getUniqueString("test_early_termination"); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/throttle/TestRouterReadQuotaThrottler.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/throttle/TestRouterReadQuotaThrottler.java index 8c2ff7aa4c6..2f2107e65f0 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/throttle/TestRouterReadQuotaThrottler.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/throttle/TestRouterReadQuotaThrottler.java @@ -12,6 +12,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.Store; @@ -50,7 +51,16 @@ public void setUp() throws Exception { Properties properties = new Properties(); properties.put(ConfigKeys.ROUTER_PER_STORE_ROUTER_QUOTA_BUFFER, 0.0); - cluster = ServiceFactory.getVeniceCluster(1, 1, numberOfRouter, 1, 10000, false, false, properties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(numberOfRouter) + .replicationFactor(1) + .partitionSize(10000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(properties) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); VersionCreationResponse response = cluster.getNewStoreVersion(); Assert.assertFalse(response.isError()); diff --git a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/DaVinciPartialKeyLookupBenchmark.java b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/DaVinciPartialKeyLookupBenchmark.java index 82e8f00fbd0..e9458752cd5 100644 --- a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/DaVinciPartialKeyLookupBenchmark.java +++ b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/DaVinciPartialKeyLookupBenchmark.java @@ -10,6 +10,7 @@ import com.linkedin.venice.client.store.ComputeGenericRecord; import com.linkedin.venice.client.store.predicate.Predicate; import com.linkedin.venice.client.store.streaming.StreamingCallback; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; @@ -76,7 +77,9 @@ public static void main(String[] args) throws Exception { @Setup public void setUp() throws Exception { Utils.thisIsLocalhost(); - cluster = getVeniceCluster(1, 1, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = getVeniceCluster(options); String storeName = buildVectorStore(cluster); client = getGenericAvroDaVinciClient( diff --git a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/IngestionBenchmarkInSingleProcess.java b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/IngestionBenchmarkInSingleProcess.java index 84fb77dbfcc..fe2b70779aa 100644 --- a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/IngestionBenchmarkInSingleProcess.java +++ b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/IngestionBenchmarkInSingleProcess.java @@ -5,6 +5,7 @@ import com.linkedin.davinci.client.DaVinciClient; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; @@ -61,7 +62,9 @@ public class IngestionBenchmarkInSingleProcess { @Setup public void setUp() throws Exception { Utils.thisIsLocalhost(); - cluster = ServiceFactory.getVeniceCluster(1, 1, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = ServiceFactory.getVeniceCluster(options); if (valueType.equals("FLOAT_VECTOR")) { storeName = buildFloatVectorStore(cluster, Integer.parseInt(valueLength)); diff --git a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/MixedIngestionBenchmark.java b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/MixedIngestionBenchmark.java index 154e87a6b6b..a8dd1cedf53 100644 --- a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/MixedIngestionBenchmark.java +++ b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/MixedIngestionBenchmark.java @@ -6,6 +6,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -60,7 +61,16 @@ public void setUp() throws Exception { Utils.thisIsLocalhost(); int numberOfController = 1; - cluster = ServiceFactory.getVeniceCluster(numberOfController, 0, 0, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); cluster.addVeniceServer(new Properties(getVeniceServerProperties())); // JMH benchmark relies on System.exit to finish one round of benchmark run, otherwise it will hang there. TestUtils.restoreSystemExit(); diff --git a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/VeniceClientBenchmark.java b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/VeniceClientBenchmark.java index 42afd7d86cd..03451480199 100644 --- a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/VeniceClientBenchmark.java +++ b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/VeniceClientBenchmark.java @@ -6,6 +6,7 @@ import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; @@ -55,7 +56,9 @@ public class VeniceClientBenchmark { @Setup public void setUp() throws Exception { Utils.thisIsLocalhost(); - cluster = getVeniceCluster(1, 1, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = getVeniceCluster(options); String storeName = buildStore(cluster); cluster.useControllerClient(c -> c.updateStore(storeName, new UpdateStoreQueryParams().setReadQuotaInCU(10000))); client = ClientFactory.getAndStartGenericAvroClient( diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/offsets/DeepCopyStorageMetadataService.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/offsets/DeepCopyStorageMetadataService.java index a1ea0e46e41..48538ecfca8 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/offsets/DeepCopyStorageMetadataService.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/offsets/DeepCopyStorageMetadataService.java @@ -28,9 +28,10 @@ public DeepCopyStorageMetadataService(StorageMetadataService delegate) { } @Override - public void computeStoreVersionState(String topicName, Function mapFunction) - throws VeniceException { - delegateStorageMetadataService.computeStoreVersionState(topicName, previousStoreVersionState -> { + public StoreVersionState computeStoreVersionState( + String topicName, + Function mapFunction) throws VeniceException { + return delegateStorageMetadataService.computeStoreVersionState(topicName, previousStoreVersionState -> { StoreVersionState newSVS = mapFunction.apply( previousStoreVersionState == null ? null diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/offsets/InMemoryStorageMetadataService.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/offsets/InMemoryStorageMetadataService.java index 943d1f62155..0b7ff817fc4 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/offsets/InMemoryStorageMetadataService.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/offsets/InMemoryStorageMetadataService.java @@ -19,10 +19,12 @@ public class InMemoryStorageMetadataService extends InMemoryOffsetManager implem private final ConcurrentMap topicToStoreVersionStateMap = new ConcurrentHashMap<>(); @Override - public void computeStoreVersionState(String topicName, Function mapFunction) - throws VeniceException { + public StoreVersionState computeStoreVersionState( + String topicName, + Function mapFunction) throws VeniceException { LOGGER.info("InMemoryStorageMetadataService.compute(StoreVersionState) called for topicName: {}", topicName); - topicToStoreVersionStateMap.compute(topicName, (s, storeVersionState) -> mapFunction.apply(storeVersionState)); + return topicToStoreVersionStateMap + .compute(topicName, (s, storeVersionState) -> mapFunction.apply(storeVersionState)); } @Override diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/tehuti/MockTehutiReporter.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/tehuti/MockTehutiReporter.java index 78ece3f427c..af3066ea75f 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/tehuti/MockTehutiReporter.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/tehuti/MockTehutiReporter.java @@ -18,6 +18,10 @@ public MockTehutiReporter() { @Override public void init(List metrics) { + // Ensure lately added reporter has all metrics. + for (TehutiMetric metric: metrics) { + this.metrics.put(metric.name(), metric); + } } @Override diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/unit/kafka/consumer/MockInMemoryConsumer.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/unit/kafka/consumer/MockInMemoryConsumer.java index f934e341b81..154231ab9da 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/unit/kafka/consumer/MockInMemoryConsumer.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/unit/kafka/consumer/MockInMemoryConsumer.java @@ -6,6 +6,7 @@ import com.linkedin.venice.pubsub.PubSubTopicPartitionInfo; import com.linkedin.venice.pubsub.api.PubSubConsumerAdapter; import com.linkedin.venice.pubsub.api.PubSubMessage; +import com.linkedin.venice.pubsub.api.PubSubPosition; import com.linkedin.venice.pubsub.api.PubSubTopic; import com.linkedin.venice.pubsub.api.PubSubTopicPartition; import com.linkedin.venice.pubsub.api.exceptions.PubSubUnsubscribedTopicPartitionException; @@ -61,6 +62,11 @@ public synchronized void subscribe(PubSubTopicPartition pubSubTopicPartition, lo offsets.put(pubSubTopicPartition, lastReadOffset); } + @Override + public void subscribe(PubSubTopicPartition pubSubTopicPartition, PubSubPosition lastReadPubSubPosition) { + throw new UnsupportedOperationException("Not implemented"); + } + @Override public synchronized void unSubscribe(PubSubTopicPartition pubSubTopicPartition) { delegate.unSubscribe(pubSubTopicPartition); diff --git a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/DataProviderUtils.java b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/DataProviderUtils.java index d32252d4459..f20bccd6644 100644 --- a/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/DataProviderUtils.java +++ b/internal/venice-test-common/src/main/java/com/linkedin/venice/utils/DataProviderUtils.java @@ -25,6 +25,7 @@ public class DataProviderUtils { public static final Object[] BOOLEAN = { false, true }; public static final Object[] BOOLEAN_FALSE = { false }; + public static final Object[] OPTIONAL_BOOLEAN = { false, true, null }; public static final Object[] COMPRESSION_STRATEGIES = { NO_OP, GZIP, ZSTD_WITH_DICT }; public static final Object[] PARTITION_COUNTS = { 1, 2, 3, 4, 8, 10, 16, 19, 92, 128 }; @@ -49,6 +50,11 @@ public static Object[][] twoBoolean() { return allPermutationGenerator(BOOLEAN, BOOLEAN); } + @DataProvider(name = "Boolean-and-Optional-Boolean") + public static Object[][] booleanAndOptionalBoolean() { + return allPermutationGenerator(BOOLEAN, OPTIONAL_BOOLEAN); + } + @DataProvider(name = "Three-True-and-False") public static Object[][] threeBoolean() { return allPermutationGenerator(BOOLEAN, BOOLEAN, BOOLEAN); diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/AuditInfo.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/AuditInfo.java index acc9828751b..fb4eceab532 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/AuditInfo.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/AuditInfo.java @@ -10,6 +10,7 @@ public class AuditInfo { private String url; private Map params; private String method; + private String clientIp; public AuditInfo(Request request) { this.url = request.url(); @@ -18,49 +19,31 @@ public AuditInfo(Request request) { this.params.put(param, request.queryParams(param)); } this.method = request.requestMethod(); + this.clientIp = request.ip() + ":" + request.raw().getRemotePort(); } - /** - * @return a string representation of {@link AuditInfo} object. - */ @Override public String toString() { - StringJoiner joiner = new StringJoiner(" "); - joiner.add("[AUDIT]"); - joiner.add(method); - joiner.add(url); - joiner.add(params.toString()); - return joiner.toString(); + return formatAuditMessage("[AUDIT]", null); } - /** - * @return a audit-successful string. - */ public String successString() { - return toString(true, null); + return formatAuditMessage("[AUDIT]", "SUCCESS"); } - /** - * @return a audit-failure string. - */ public String failureString(String errMsg) { - return toString(false, errMsg); + return formatAuditMessage("[AUDIT]", "FAILURE: " + (errMsg != null ? errMsg : "")); } - private String toString(boolean success, String errMsg) { - StringJoiner joiner = new StringJoiner(" "); - joiner.add("[AUDIT]"); - if (success) { - joiner.add("SUCCESS"); - } else { - joiner.add("FAILURE: "); - if (errMsg != null) { - joiner.add(errMsg); - } + private String formatAuditMessage(String prefix, String status) { + StringJoiner joiner = new StringJoiner(" ").add(prefix); + + if (status != null) { + joiner.add(status); } - joiner.add(method); - joiner.add(url); - joiner.add(params.toString()); + + joiner.add(method).add(url).add(params.toString()).add("ClientIP: " + clientIp); + return joiner.toString(); } } diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java index 8291c4117e2..c38f7baa7af 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/VeniceHelixAdmin.java @@ -7,6 +7,7 @@ import static com.linkedin.venice.ConfigKeys.SSL_KAFKA_BOOTSTRAP_SERVERS; import static com.linkedin.venice.ConfigKeys.SSL_TO_KAFKA_LEGACY; import static com.linkedin.venice.controller.UserSystemStoreLifeCycleHelper.AUTO_META_SYSTEM_STORE_PUSH_ID_PREFIX; +import static com.linkedin.venice.exceptions.VeniceNoStoreException.DOES_NOT_EXISTS; import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_HYBRID_OFFSET_LAG_THRESHOLD; import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_HYBRID_TIME_LAG_THRESHOLD; import static com.linkedin.venice.meta.HybridStoreConfigImpl.DEFAULT_REAL_TIME_TOPIC_NAME; @@ -3894,15 +3895,44 @@ public boolean isRTTopicDeletionPermittedByAllControllers(String clusterName, St String rtTopicName = Utils.composeRealTimeTopic(storeName); Map controllerClientMap = getControllerClientMap(clusterName); for (Map.Entry controllerClientEntry: controllerClientMap.entrySet()) { - StoreResponse storeResponse = controllerClientEntry.getValue().getStore(storeName); - if (storeResponse.isError()) { + StoreResponse storeResponse; + try { + storeResponse = RetryUtils.executeWithMaxAttemptAndExponentialBackoff(() -> { + StoreResponse response = controllerClientEntry.getValue().getStore(storeName); + if (response.isError() && response.getError().contains(DOES_NOT_EXISTS)) { + throw new VeniceNoStoreException(storeName); + } + return response; + }, + 5, + Duration.ofMillis(10), + Duration.ofMillis(500), + Duration.ofSeconds(5), + Collections.singletonList(VeniceNoStoreException.class)); + if (storeResponse.isError()) { + if (storeResponse.getError().contains(DOES_NOT_EXISTS)) { + LOGGER.warn( + "Store {} does not exist in cluster {} in fabric {}, probably deleted already, skipping RT check", + storeName, + clusterName, + controllerClientEntry.getKey()); + continue; + } + LOGGER.warn( + "Skipping RT cleanup check for store: {} in cluster: {} due to unable to get store from fabric: {} Error: {}", + storeName, + clusterName, + controllerClientEntry.getKey(), + storeResponse.getError()); + return false; + } + } catch (VeniceNoStoreException e) { LOGGER.warn( - "Skipping RT cleanup check for store: {} in cluster: {} due to unable to get store from fabric: {} Error: {}", + "Store {} does not exist in cluster {} in fabric {}, probably deleted already, skipping RT check", storeName, clusterName, - controllerClientEntry.getKey(), - storeResponse.getError()); - return false; + controllerClientEntry.getKey()); + continue; } if (Version.containsHybridVersion(storeResponse.getStore().getVersions())) { LOGGER.warn( diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/ControllerRoutes.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/ControllerRoutes.java index 6f6c7aa9180..c2fa65fe4fb 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/ControllerRoutes.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/ControllerRoutes.java @@ -121,7 +121,7 @@ public Route updateKafkaTopicLogCompaction(Admin admin) { return updateKafkaTopicConfig(admin, adminRequest -> { AdminSparkServer.validateParams(adminRequest, UPDATE_KAFKA_TOPIC_LOG_COMPACTION.getParams(), admin); PubSubTopic topicName = pubSubTopicRepository.getTopic(adminRequest.queryParams(TOPIC)); - boolean kafkaTopicLogCompactionEnabled = Utils.parseBooleanFromString( + boolean kafkaTopicLogCompactionEnabled = Utils.parseBooleanOrThrow( adminRequest.queryParams(KAFKA_TOPIC_LOG_COMPACTION_ENABLED), KAFKA_TOPIC_LOG_COMPACTION_ENABLED); diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/CreateVersion.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/CreateVersion.java index f2bbdcaa16f..439fdf8a8d2 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/CreateVersion.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/CreateVersion.java @@ -82,20 +82,19 @@ protected static void extractOptionalParamsFromRequestTopicRequest( request.setPartitioners(httpRequest.queryParamOrDefault(PARTITIONERS, null)); request.setSendStartOfPush( - Utils.parseBooleanFromString(httpRequest.queryParamOrDefault(SEND_START_OF_PUSH, "false"), SEND_START_OF_PUSH)); + Utils.parseBooleanOrThrow(httpRequest.queryParamOrDefault(SEND_START_OF_PUSH, "false"), SEND_START_OF_PUSH)); request.setSorted( - Utils.parseBooleanFromString( - httpRequest.queryParamOrDefault(PUSH_IN_SORTED_ORDER, "false"), - PUSH_IN_SORTED_ORDER)); + Utils + .parseBooleanOrThrow(httpRequest.queryParamOrDefault(PUSH_IN_SORTED_ORDER, "false"), PUSH_IN_SORTED_ORDER)); request.setWriteComputeEnabled( - Utils.parseBooleanFromString( + Utils.parseBooleanOrThrow( httpRequest.queryParamOrDefault(IS_WRITE_COMPUTE_ENABLED, "false"), IS_WRITE_COMPUTE_ENABLED)); request.setSeparateRealTimeTopicEnabled( - Utils.parseBooleanFromString( + Utils.parseBooleanOrThrow( httpRequest.queryParamOrDefault(SEPARATE_REAL_TIME_TOPIC_ENABLED, "false"), SEPARATE_REAL_TIME_TOPIC_ENABLED)); @@ -109,7 +108,7 @@ protected static void extractOptionalParamsFromRequestTopicRequest( * Version level override to defer marking this new version to the serving version post push completion. */ request.setDeferVersionSwap( - Utils.parseBooleanFromString(httpRequest.queryParamOrDefault(DEFER_VERSION_SWAP, "false"), DEFER_VERSION_SWAP)); + Utils.parseBooleanOrThrow(httpRequest.queryParamOrDefault(DEFER_VERSION_SWAP, "false"), DEFER_VERSION_SWAP)); request.setTargetedRegions(httpRequest.queryParamOrDefault(TARGETED_REGIONS, null)); diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/DataRecoveryRoutes.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/DataRecoveryRoutes.java index 55bc6c3b352..2dde4c8fc36 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/DataRecoveryRoutes.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/DataRecoveryRoutes.java @@ -48,12 +48,11 @@ public void internalHandle(Request request, ControllerResponse veniceResponse) { int version = Utils.parseIntFromString(request.queryParams(VERSION), VERSION); String sourceFabric = request.queryParams(SOURCE_FABRIC); String destinationFabric = request.queryParams(FABRIC); - boolean copyAllVersionConfigs = Utils.parseBooleanFromString( + boolean copyAllVersionConfigs = Utils.parseBooleanOrThrow( request.queryParams(DATA_RECOVERY_COPY_ALL_VERSION_CONFIGS), DATA_RECOVERY_COPY_ALL_VERSION_CONFIGS); - boolean sourceVersionIncluded = Utils.parseBooleanFromString( - request.queryParams(SOURCE_FABRIC_VERSION_INCLUDED), - SOURCE_FABRIC_VERSION_INCLUDED); + boolean sourceVersionIncluded = Utils + .parseBooleanOrThrow(request.queryParams(SOURCE_FABRIC_VERSION_INCLUDED), SOURCE_FABRIC_VERSION_INCLUDED); Optional sourceVersion; if (sourceVersionIncluded) { Version sourceVersionObject = null; diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/RoutersClusterConfigRoutes.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/RoutersClusterConfigRoutes.java index 36450c97021..41ae71d83f8 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/RoutersClusterConfigRoutes.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/RoutersClusterConfigRoutes.java @@ -37,7 +37,7 @@ public void internalHandle(Request request, ControllerResponse veniceResponse) { AdminSparkServer.validateParams(request, ENABLE_THROTTLING.getParams(), admin); String clusterName = request.queryParams(CLUSTER); veniceResponse.setCluster(clusterName); - boolean status = Utils.parseBooleanFromString(request.queryParams(STATUS), "enableThrottling"); + boolean status = Utils.parseBooleanOrThrow(request.queryParams(STATUS), "enableThrottling"); admin.updateRoutersClusterConfig( clusterName, Optional.of(status), @@ -62,7 +62,7 @@ public void internalHandle(Request request, ControllerResponse veniceResponse) { AdminSparkServer.validateParams(request, ENABLE_MAX_CAPACITY_PROTECTION.getParams(), admin); String clusterName = request.queryParams(CLUSTER); veniceResponse.setCluster(clusterName); - boolean status = Utils.parseBooleanFromString(request.queryParams(STATUS), "enableMaxCapacityProtection"); + boolean status = Utils.parseBooleanOrThrow(request.queryParams(STATUS), "enableMaxCapacityProtection"); admin.updateRoutersClusterConfig( clusterName, Optional.empty(), @@ -87,7 +87,7 @@ public void internalHandle(Request request, ControllerResponse veniceResponse) { AdminSparkServer.validateParams(request, ENABLE_QUOTA_REBALANCED.getParams(), admin); String clusterName = request.queryParams(CLUSTER); veniceResponse.setCluster(clusterName); - boolean status = Utils.parseBooleanFromString(request.queryParams(STATUS), "enableQuotaRebalance"); + boolean status = Utils.parseBooleanOrThrow(request.queryParams(STATUS), "enableQuotaRebalance"); int expectedRouterCount = Utils.parseIntFromString(request.queryParams(EXPECTED_ROUTER_COUNT), "expectedRouterCount"); admin.updateRoutersClusterConfig( diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/SkipAdminRoute.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/SkipAdminRoute.java index 295953e3a15..107c36910ba 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/SkipAdminRoute.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/SkipAdminRoute.java @@ -39,7 +39,7 @@ public Route skipAdminMessage(Admin admin) { AdminSparkServer.validateParams(request, SKIP_ADMIN.getParams(), admin); responseObject.setCluster(request.queryParams(CLUSTER)); long offset = Utils.parseLongFromString(request.queryParams(OFFSET), OFFSET); - boolean skipDIV = Utils.parseBooleanFromString(request.queryParams(SKIP_DIV), SKIP_DIV); + boolean skipDIV = Utils.parseBooleanOrThrow(request.queryParams(SKIP_DIV), SKIP_DIV); admin.skipAdminMessage(responseObject.getCluster(), offset, skipDIV); } catch (Throwable e) { responseObject.setError(e); diff --git a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java index ccce9bcc912..03e08113bbb 100644 --- a/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java +++ b/services/venice-controller/src/main/java/com/linkedin/venice/controller/server/StoresRoutes.java @@ -520,7 +520,7 @@ public void internalHandle(Request request, TrackableControllerResponse veniceRe String clusterName = request.queryParams(CLUSTER); String storeName = request.queryParams(NAME); boolean abortMigratingStore = - Utils.parseBooleanFromString(request.queryParams(IS_ABORT_MIGRATION_CLEANUP), IS_ABORT_MIGRATION_CLEANUP); + Utils.parseBooleanOrFalse(request.queryParams(IS_ABORT_MIGRATION_CLEANUP), IS_ABORT_MIGRATION_CLEANUP); veniceResponse.setCluster(clusterName); veniceResponse.setName(storeName); @@ -696,7 +696,7 @@ public void internalHandle(Request request, ControllerResponse veniceResponse) { String cluster = request.queryParams(CLUSTER); String storeName = request.queryParams(NAME); String operation = request.queryParams(OPERATION); - boolean status = Utils.parseBooleanFromString(request.queryParams(STATUS), "storeAccessStatus"); + boolean status = Utils.parseBooleanOrThrow(request.queryParams(STATUS), "storeAccessStatus"); veniceResponse.setCluster(cluster); veniceResponse.setName(storeName); @@ -812,7 +812,7 @@ public void internalHandle(Request request, ControllerResponse veniceResponse) { String cluster = request.queryParams(CLUSTER); boolean enableActiveActiveReplicationForCluster = - Utils.parseBooleanFromString(request.queryParams(STATUS), STATUS); + Utils.parseBooleanOrThrow(request.queryParams(STATUS), STATUS); String regionsFilterParams = request.queryParamOrDefault(REGIONS_FILTER, null); admin.configureActiveActiveReplication( diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/AuditInfoTest.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/AuditInfoTest.java new file mode 100644 index 00000000000..4d80e1de2f9 --- /dev/null +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/AuditInfoTest.java @@ -0,0 +1,94 @@ +package com.linkedin.venice.controller; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import java.util.HashSet; +import java.util.Set; +import javax.servlet.http.HttpServletRequest; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import spark.Request; + + +public class AuditInfoTest { + private static final String TEST_URL = "http://localhost/test"; + private static final String METHOD_GET = "GET"; + private static final String CLIENT_IP = "127.0.0.1"; + private static final int CLIENT_PORT = 8080; + private static final String PARAM_1 = "param1"; + private static final String PARAM_2 = "param2"; + private static final String VALUE_1 = "value1"; + private static final String VALUE_2 = "value2"; + private static final String AUDIT_PREFIX = "[AUDIT]"; + private static final String SUCCESS = "SUCCESS"; + private static final String FAILURE = "FAILURE"; + private static final String ERROR_MESSAGE = "Some error"; + + private Request request; + private AuditInfo auditInfo; + private HttpServletRequest httpServletRequest; + + @BeforeMethod + public void setUp() { + request = mock(Request.class); + when(request.url()).thenReturn(TEST_URL); + when(request.requestMethod()).thenReturn(METHOD_GET); + when(request.ip()).thenReturn(CLIENT_IP); + + Set queryParams = new HashSet<>(); + queryParams.add(PARAM_1); + queryParams.add(PARAM_2); + + when(request.queryParams()).thenReturn(queryParams); + when(request.queryParams(PARAM_1)).thenReturn(VALUE_1); + when(request.queryParams(PARAM_2)).thenReturn(VALUE_2); + + httpServletRequest = mock(HttpServletRequest.class); + when(httpServletRequest.getRemotePort()).thenReturn(CLIENT_PORT); + when(request.raw()).thenReturn(httpServletRequest); + + auditInfo = new AuditInfo(request); + } + + @Test + public void testToStringReturnsExpectedFormat() { + String result = auditInfo.toString(); + assertTrue(result.contains(AUDIT_PREFIX)); + assertTrue(result.contains(METHOD_GET)); + assertTrue(result.contains(TEST_URL)); + assertTrue(result.contains(PARAM_1 + "=" + VALUE_1)); + assertTrue(result.contains(PARAM_2 + "=" + VALUE_2)); + assertTrue(result.contains("ClientIP: " + CLIENT_IP + ":" + CLIENT_PORT)); + } + + @Test + public void testSuccessStringReturnsExpectedFormat() { + String result = auditInfo.successString(); + assertTrue(result.contains(AUDIT_PREFIX)); + assertTrue(result.contains(SUCCESS)); + assertTrue(result.contains(METHOD_GET)); + assertTrue(result.contains(TEST_URL)); + assertTrue(result.contains("ClientIP: " + CLIENT_IP)); + } + + @Test + public void testFailureStringReturnsExpectedFormat() { + String result = auditInfo.failureString(ERROR_MESSAGE); + assertTrue(result.contains(AUDIT_PREFIX)); + assertTrue(result.contains(FAILURE)); + assertTrue(result.contains(ERROR_MESSAGE)); + assertTrue(result.contains(METHOD_GET)); + assertTrue(result.contains(TEST_URL)); + assertTrue(result.contains("ClientIP: " + CLIENT_IP)); + } + + @Test + public void testFailureStringHandlesNullErrorMessage() { + String result = auditInfo.failureString(null); + assertTrue(result.contains(AUDIT_PREFIX)); + assertFalse(result.contains("null")); + } +} diff --git a/services/venice-controller/src/test/java/com/linkedin/venice/controller/server/StoreRoutesTest.java b/services/venice-controller/src/test/java/com/linkedin/venice/controller/server/StoreRoutesTest.java index 6514814dd93..27a627d1877 100644 --- a/services/venice-controller/src/test/java/com/linkedin/venice/controller/server/StoreRoutesTest.java +++ b/services/venice-controller/src/test/java/com/linkedin/venice/controller/server/StoreRoutesTest.java @@ -101,7 +101,6 @@ public void testDeleteStore() throws Exception { Request request = mock(Request.class); doReturn(TEST_CLUSTER).when(request).queryParams(eq(ControllerApiConstants.CLUSTER)); doReturn(TEST_STORE_NAME).when(request).queryParams(eq(ControllerApiConstants.NAME)); - doReturn("true").when(request).queryParams(eq(ControllerApiConstants.IS_ABORT_MIGRATION_CLEANUP)); Route deleteStoreRoute = new StoresRoutes(false, Optional.empty(), pubSubTopicRepository).deleteStore(mockAdmin); TrackableControllerResponse trackableControllerResponse = ObjectMapperFactory.getInstance() @@ -112,6 +111,7 @@ public void testDeleteStore() throws Exception { Assert.assertEquals(trackableControllerResponse.getCluster(), TEST_CLUSTER); Assert.assertEquals(trackableControllerResponse.getName(), TEST_STORE_NAME); + doReturn("true").when(request).queryParams(eq(ControllerApiConstants.IS_ABORT_MIGRATION_CLEANUP)); String errMessage = "Store " + TEST_STORE_NAME + "'s migrating flag is false. Not safe to delete a store " + "that is assumed to be migrating without the migrating flag setup as true."; doThrow(new VeniceException(errMessage, INVALID_CONFIG)).when(mockAdmin) diff --git a/settings.gradle b/settings.gradle index 834f01adb53..340d0830673 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,10 +6,6 @@ pluginManagement { url "https://linkedin.jfrog.io/artifactory/open-source" } gradlePluginPortal() - maven { - // Needed for DuckDB SNAPSHOT. TODO: Remove when the real release is published! - url = uri('https://oss.sonatype.org/content/repositories/snapshots/') - } } }