From 4a555eaa0d8f265d34d064d13acefe2fb302bf10 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 20 Jan 2022 13:22:35 +0800 Subject: [PATCH 001/103] Bump Pulsar 2.9.1 and Kafka client 3.0.0 (#238) --- driver-kafka/pom.xml | 2 +- driver-pulsar/deploy/ssd/deploy_with_stats.yaml | 2 +- driver-pulsar/deploy/ssd/terraform.tfvars | 1 + driver-pulsar/pom.xml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index a44de0b86..83744ea7c 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -39,7 +39,7 @@ org.apache.kafka kafka-clients - 2.8.1 + 3.0.0 diff --git a/driver-pulsar/deploy/ssd/deploy_with_stats.yaml b/driver-pulsar/deploy/ssd/deploy_with_stats.yaml index 146b2ddb2..34726d9c2 100644 --- a/driver-pulsar/deploy/ssd/deploy_with_stats.yaml +++ b/driver-pulsar/deploy/ssd/deploy_with_stats.yaml @@ -121,7 +121,7 @@ zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" serviceUrl: "pulsar://{{ hostvars[groups['pulsar'][0]].private_ip }}:6650/" httpUrl: "http://{{ hostvars[groups['pulsar'][0]].private_ip }}:8080/" - pulsarVersion: "2.8.1" + pulsarVersion: "2.9.1" - file: path=/opt/pulsar state=absent - file: path=/opt/pulsar state=directory - name: Download Pulsar binary package diff --git a/driver-pulsar/deploy/ssd/terraform.tfvars b/driver-pulsar/deploy/ssd/terraform.tfvars index 6da7404a6..4de684d91 100644 --- a/driver-pulsar/deploy/ssd/terraform.tfvars +++ b/driver-pulsar/deploy/ssd/terraform.tfvars @@ -1,5 +1,6 @@ public_key_path = "~/.ssh/pulsar_aws.pub" region = "us-west-2" +az = "us-west-2a" ami = "ami-9fa343e7" // RHEL-7.4 instance_types = { diff --git a/driver-pulsar/pom.xml b/driver-pulsar/pom.xml index 5a0939551..768c6c57a 100644 --- a/driver-pulsar/pom.xml +++ b/driver-pulsar/pom.xml @@ -30,7 +30,7 @@ driver-pulsar - 2.8.1 + 2.9.1 From 3b793a27855c156af98c918fada9a314c1a81c23 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Thu, 20 Jan 2022 21:25:02 +0800 Subject: [PATCH 002/103] Set flushEntrylogBytes (#240) --- driver-pulsar/deploy/ssd/templates/bookkeeper.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/driver-pulsar/deploy/ssd/templates/bookkeeper.conf b/driver-pulsar/deploy/ssd/templates/bookkeeper.conf index 8140634a9..1d204a6eb 100644 --- a/driver-pulsar/deploy/ssd/templates/bookkeeper.conf +++ b/driver-pulsar/deploy/ssd/templates/bookkeeper.conf @@ -29,6 +29,13 @@ ledgerDirectories=/mnt/storage/0,/mnt/storage/1 journalSyncData=true journalWriteData=true +# Entry log flush interval in bytes. +# Default is 0. 0 or less disables this feature and effectively flush +# happens on log rotation. +# Flushing in smaller chunks but more frequently reduces spikes in disk +# I/O. Flushing too frequently may also affect performance negatively. +flushEntrylogBytes=67108864 + ## Regular Bookie settings # Port that bookie server listen on From ddd4b900918451ad3caee46bf1fd490a1ba62b25 Mon Sep 17 00:00:00 2001 From: Elad Leev Date: Mon, 14 Feb 2022 03:11:31 +0200 Subject: [PATCH 003/103] Fix missing dash (#244) --- deployment/kubernetes/helm/benchmark/templates/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml b/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml index aa440dc3a..1ff03e12a 100644 --- a/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml +++ b/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml @@ -40,7 +40,7 @@ spec: - name: WORKERS value: '{{ template "workers" .Values }}' - name: HEAP_OPTS - value: 'Xms4G -Xmx4G' + value: '-Xms4G -Xmx4G' command: ["sh", "-c"] args: - > From 8c116f5e0beb31a918ab3c371b4d7d6177eed681 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Mon, 7 Mar 2022 21:13:05 +0800 Subject: [PATCH 004/103] Use the latest apache pulsar grafana dashboard (#252) --- driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service b/driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service index cd73422e8..24e69b7f7 100644 --- a/driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service +++ b/driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service @@ -8,7 +8,7 @@ Requires=prometheus.service [Service] WorkingDirectory=/opt/pulsar ExecStartPre=/usr/bin/docker pull streamnative/apache-pulsar-grafana-dashboard:latest -ExecStart=/usr/bin/docker run --restart=always --name=systemd_pulsar_dashboard -p3000:3000 -e PULSAR_PROMETHEUS_URL=http://{{ hostvars[groups['prometheus'][0]].private_ip }}:9090/ -e PULSAR_CLUSTER=local streamnative/apache-pulsar-grafana-dashboard:0.0.8 +ExecStart=/usr/bin/docker run --restart=always --name=systemd_pulsar_dashboard -p3000:3000 -e PULSAR_PROMETHEUS_URL=http://{{ hostvars[groups['prometheus'][0]].private_ip }}:9090/ -e PULSAR_CLUSTER=local streamnative/apache-pulsar-grafana-dashboard:latest ExecStop=/usr/bin/docker stop systemd_pulsar_dashboard ExecStopPost=/usr/bin/docker rm -f systemd_pulsar_dashboard ExecReload=/usr/bin/docker restart systemd_pulsar_dashboard From 44293d5c1d46d6486b3d5e5631d6984a927800e8 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 10 Mar 2022 09:05:59 +0100 Subject: [PATCH 005/103] Load balance topics on RabbitMQ nodes (#239) * Load balance topics on RabbitMQ nodes * Replace isOpen() checks by catching AlreadyClosedException --- .../benchmark/worker/LocalWorker.java | 2 +- .../worker/commands/TopicSubscription.java | 7 ++ driver-rabbitmq/deploy/deploy.yaml | 9 +- .../deploy/templates/rabbitmq.yaml | 30 +++++ driver-rabbitmq/rabbitmq.yaml | 3 +- .../rabbitmq/RabbitMqBenchmarkConsumer.java | 14 ++- .../rabbitmq/RabbitMqBenchmarkDriver.java | 113 ++++++++++++------ .../rabbitmq/RabbitMqBenchmarkProducer.java | 9 +- .../driver/rabbitmq/RabbitMqConfig.java | 7 +- 9 files changed, 143 insertions(+), 51 deletions(-) create mode 100644 driver-rabbitmq/deploy/templates/rabbitmq.yaml diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index 5c5ad560b..022f4fa2e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -143,10 +143,10 @@ public List createTopics(TopicsInfo topicsInfo) { Timer timer = new Timer(); - String topicPrefix = benchmarkDriver.getTopicNamePrefix(); List topics = new ArrayList<>(); for (int i = 0; i < topicsInfo.numberOfTopics; i++) { + String topicPrefix = benchmarkDriver.getTopicNamePrefix(); String topic = String.format("%s-%s-%04d", topicPrefix, RandomGenerator.getRandomString(), i); topics.add(topic); futures.add(benchmarkDriver.createTopic(topic, topicsInfo.numberOfPartitionsPerTopic)); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java index 8dc0d7d30..de35a5211 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java @@ -30,4 +30,11 @@ public TopicSubscription(String topic, String subscription) { this.subscription = subscription; } + @Override + public String toString() { + return "TopicSubscription{" + + "topic='" + topic + '\'' + + ", subscription='" + subscription + '\'' + + '}'; + } } diff --git a/driver-rabbitmq/deploy/deploy.yaml b/driver-rabbitmq/deploy/deploy.yaml index 37fae4cfd..2d37bca0d 100644 --- a/driver-rabbitmq/deploy/deploy.yaml +++ b/driver-rabbitmq/deploy/deploy.yaml @@ -173,11 +173,10 @@ src: "templates/workers.yaml" dest: "/opt/benchmark/workers.yaml" - - name: Configure service URL - lineinfile: - dest: /opt/benchmark/driver-rabbitmq/rabbitmq.yaml - regexp: '^brokerAddress\: ' - line: "brokerAddress: {{ hostvars[groups['rabbitmq'][0]].private_ip }}" + - template: + src: "templates/rabbitmq.yaml" + dest: "/opt/benchmark/driver-rabbitmq/rabbitmq.yaml" + - name: Configure benchmark-worker memory lineinfile: dest: /opt/benchmark/bin/benchmark-worker diff --git a/driver-rabbitmq/deploy/templates/rabbitmq.yaml b/driver-rabbitmq/deploy/templates/rabbitmq.yaml new file mode 100644 index 000000000..e181768c8 --- /dev/null +++ b/driver-rabbitmq/deploy/templates/rabbitmq.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +name: RabbitMQ +driverClass: io.openmessaging.benchmark.driver.rabbitmq.RabbitMqBenchmarkDriver + +# RabbitMq client specific configurations + +amqpUris: +{% for pulsar in groups['rabbitmq'] %} + - amqp://admin:admin@{{ hostvars[pulsar].private_ip }}:5672 +{% endfor %} +messagePersistence: false \ No newline at end of file diff --git a/driver-rabbitmq/rabbitmq.yaml b/driver-rabbitmq/rabbitmq.yaml index 7ba41ff65..7651b6425 100644 --- a/driver-rabbitmq/rabbitmq.yaml +++ b/driver-rabbitmq/rabbitmq.yaml @@ -23,5 +23,6 @@ driverClass: io.openmessaging.benchmark.driver.rabbitmq.RabbitMqBenchmarkDriver # RabbitMq client specific configurations -brokerAddress: localhost +amqpUris: + - amqp://localhost messagePersistence: false \ No newline at end of file diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java index 95f8d32ec..4120f5cf9 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java @@ -18,6 +18,7 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; +import com.rabbitmq.client.AlreadyClosedException; import java.io.IOException; import com.rabbitmq.client.AMQP.BasicProperties; @@ -27,9 +28,13 @@ import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.ConsumerCallback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RabbitMqBenchmarkConsumer extends DefaultConsumer implements BenchmarkConsumer { + private static final Logger log = LoggerFactory.getLogger(RabbitMqBenchmarkConsumer.class); + private final Channel channel; private final ConsumerCallback callback; @@ -42,15 +47,16 @@ public RabbitMqBenchmarkConsumer(Channel channel, String queueName, ConsumerCall } @Override - public void handleDelivery(String consumerTag, Envelope envelope, BasicProperties properties, byte[] body) - throws IOException { + public void handleDelivery(String consumerTag, Envelope envelope, BasicProperties properties, byte[] body) { callback.messageReceived(body, properties.getTimestamp().getTime()); } @Override public void close() throws Exception { - if (this.channel.isOpen()) { - this.channel.close(); + try { + channel.close(); + } catch (AlreadyClosedException e) { + log.warn("Channel already closed", e); } } diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index 5f9913da2..6b411fc5e 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -18,12 +18,18 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; +import com.rabbitmq.client.AlreadyClosedException; import java.io.File; import java.io.IOException; +import java.net.URI; import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -33,46 +39,48 @@ import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; +import io.netty.handler.codec.http.QueryStringDecoder; import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.BenchmarkDriver; import io.openmessaging.benchmark.driver.BenchmarkProducer; import io.openmessaging.benchmark.driver.ConsumerCallback; import org.apache.bookkeeper.stats.StatsLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RabbitMqBenchmarkDriver implements BenchmarkDriver { private RabbitMqConfig config; + private final AtomicInteger uriIndex = new AtomicInteger(); + private final Map connections = new ConcurrentHashMap<>(); - private Connection connection; @Override public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException { config = mapper.readValue(configurationFile, RabbitMqConfig.class); - - ConnectionFactory connectionFactory = new ConnectionFactory(); - connectionFactory.setAutomaticRecoveryEnabled(true); - connectionFactory.setHost(config.brokerAddress); - connectionFactory.setUsername("admin"); - connectionFactory.setPassword("admin"); - - try { - connection = connectionFactory.newConnection(); - } catch (TimeoutException e) { - e.printStackTrace(); - } } - - @Override - public void close() throws Exception { - if (connection != null && connection.isOpen()) { - connection.close(); + public void close() { + for(Iterator> it = connections.entrySet().iterator(); it.hasNext(); ) { + Connection connection = it.next().getValue(); + try { + connection.close(); + } catch (AlreadyClosedException e) { + log.warn("Connection already closed", e); + } catch (Exception e) { + log.error("Couldn't close connection", e); + } + it.remove(); } + } @Override public String getTopicNamePrefix() { - return "test-topic"; + // Do a round-robin on AMQP URIs + URI configUri = URI.create(config.amqpUris.get(uriIndex.getAndIncrement() % config.amqpUris.size())); + URI topicUri = configUri.resolve(configUri.getRawPath() + "?exchange=test-exchange"); + return topicUri.toString(); } @Override @@ -89,15 +97,21 @@ public CompletableFuture createTopic(String topic, int partitions) { @Override public CompletableFuture createProducer(String topic) { - Channel channel = null; - try { - channel = connection.createChannel(); - channel.exchangeDeclare(topic, BuiltinExchangeType.FANOUT, true); - channel.confirmSelect(); - } catch (IOException e) { - e.printStackTrace(); - } - return CompletableFuture.completedFuture(new RabbitMqBenchmarkProducer(channel, topic, config.messagePersistence)); + CompletableFuture future = new CompletableFuture<>(); + ForkJoinPool.commonPool().execute(() -> { + try { + String uri = topic.split("\\?")[0]; + Connection connection = getOrCreateConnection(uri); + Channel channel = connection.createChannel(); + String exchange = getExchangeName(topic); + channel.exchangeDeclare(exchange, BuiltinExchangeType.FANOUT, true); + channel.confirmSelect(); + future.complete(new RabbitMqBenchmarkProducer(channel, exchange, config.messagePersistence)); + } catch (Exception e) { + future.completeExceptionally(e); + } + }); + return future; } @Override @@ -107,17 +121,15 @@ public CompletableFuture createConsumer(String topic, String CompletableFuture future = new CompletableFuture<>(); ForkJoinPool.commonPool().execute(() -> { try { - String queueName = topic + "-" + subscriptionName; - Channel channel = null; - try { - channel = connection.createChannel(); - channel.exchangeDeclare(topic, BuiltinExchangeType.FANOUT, true); - } catch (IOException e) { - e.printStackTrace(); - } + String uri = topic.split("\\?")[0]; + Connection connection = getOrCreateConnection(uri); + Channel channel = connection.createChannel(); + String exchange = getExchangeName(topic); + String queueName = exchange + "-" + subscriptionName; + channel.exchangeDeclare(exchange, BuiltinExchangeType.FANOUT, true); // Create the queue channel.queueDeclare(queueName, true, false, false, Collections.emptyMap()); - channel.queueBind(queueName, topic, ""); + channel.queueBind(queueName, exchange, ""); future.complete(new RabbitMqBenchmarkConsumer(channel, queueName, consumerCallback)); } catch (IOException e) { future.completeExceptionally(e); @@ -127,6 +139,31 @@ public CompletableFuture createConsumer(String topic, String return future; } + private String getExchangeName(String uri) { + QueryStringDecoder decoder = new QueryStringDecoder(uri); + Map> parameters = decoder.parameters(); + + if(!parameters.containsKey("exchange")) { + throw new IllegalArgumentException("Missing exchange param"); + } + return parameters.get("exchange").get(0); + } + + private Connection getOrCreateConnection(String uri) { + return connections.computeIfAbsent(uri, uriKey -> { + try { + ConnectionFactory connectionFactory = new ConnectionFactory(); + connectionFactory.setAutomaticRecoveryEnabled(true); + connectionFactory.setUri(uri); + return connectionFactory.newConnection(); + } catch (Exception e) { + throw new RuntimeException("Couldn't establish connection", e); + } + }); + } + private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + private static final Logger log = LoggerFactory.getLogger(RabbitMqBenchmarkDriver.class); } diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java index ed58966ae..bbd117e02 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java @@ -18,6 +18,7 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; +import com.rabbitmq.client.AlreadyClosedException; import com.rabbitmq.client.ConfirmListener; import java.util.Collections; import java.util.Date; @@ -32,9 +33,13 @@ import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.concurrent.ConcurrentHashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RabbitMqBenchmarkProducer implements BenchmarkProducer { + private static final Logger log = LoggerFactory.getLogger(RabbitMqBenchmarkProducer.class); + private final Channel channel; private final String exchange; private final ConfirmListener listener; @@ -104,9 +109,11 @@ public void handleAck(long deliveryTag, boolean multiple) { @Override public void close() throws Exception { - if (channel.isOpen()) { + try { channel.removeConfirmListener(listener); channel.close(); + } catch (AlreadyClosedException e) { + log.warn("Channel already closed", e); } } diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java index d632ba153..14b3d7155 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java @@ -18,7 +18,12 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; +import java.util.ArrayList; +import java.util.List; + public class RabbitMqConfig { - public String brokerAddress; + + public List amqpUris = new ArrayList<>(); + public boolean messagePersistence = false; } From 73081cf96fd9d545ed8392dee3b7bc8c47ab3a52 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 10 Mar 2022 16:06:23 +0800 Subject: [PATCH 006/103] Create a benchmark driver for KoP (https://github.com/streamnative/kop) (#245) --- benchmark-framework/pom.xml | 6 + .../driver/kafka/KafkaBenchmarkConsumer.java | 14 +- driver-kop/kafka_to_kafka.yaml | 44 ++++ driver-kop/kafka_to_pulsar.yaml | 36 +++ driver-kop/pom.xml | 66 ++++++ driver-kop/pulsar_to_kafka.yaml | 36 +++ .../driver/kop/KopBenchmarkDriver.java | 221 ++++++++++++++++++ .../driver/kop/config/ClientType.java | 26 +++ .../benchmark/driver/kop/config/Config.java | 50 ++++ .../driver/kop/config/PulsarConfig.java | 37 +++ .../driver/kop/KopBenchmarkDriverTest.java | 106 +++++++++ driver-kop/src/test/resources/kop_custom.yaml | 47 ++++ .../src/test/resources/kop_required.yaml | 34 +++ .../resources/kop_wrong_kafka_config.yaml | 26 +++ pom.xml | 1 + 15 files changed, 748 insertions(+), 2 deletions(-) create mode 100644 driver-kop/kafka_to_kafka.yaml create mode 100644 driver-kop/kafka_to_pulsar.yaml create mode 100644 driver-kop/pom.xml create mode 100644 driver-kop/pulsar_to_kafka.yaml create mode 100644 driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java create mode 100644 driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java create mode 100644 driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java create mode 100644 driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java create mode 100644 driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java create mode 100644 driver-kop/src/test/resources/kop_custom.yaml create mode 100644 driver-kop/src/test/resources/kop_required.yaml create mode 100644 driver-kop/src/test/resources/kop_wrong_kafka_config.yaml diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index d559e0595..a379ef4ee 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -110,6 +110,12 @@ ${project.version} + + ${project.groupId} + driver-kop + ${project.version} + + ${project.groupId} driver-pravega diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java index 2f3dd695a..61c2dc479 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java @@ -48,14 +48,24 @@ public class KafkaBenchmarkConsumer implements BenchmarkConsumer { private final Future consumerTask; private volatile boolean closing = false; private boolean autoCommit; - public KafkaBenchmarkConsumer(KafkaConsumer consumer, Properties consumerConfig, ConsumerCallback callback) { + + public KafkaBenchmarkConsumer(KafkaConsumer consumer, + Properties consumerConfig, + ConsumerCallback callback) { + this(consumer, consumerConfig, callback, 100L); + } + + public KafkaBenchmarkConsumer(KafkaConsumer consumer, + Properties consumerConfig, + ConsumerCallback callback, + long pollTimeoutMs) { this.consumer = consumer; this.executor = Executors.newSingleThreadExecutor(); this.autoCommit= Boolean.valueOf((String)consumerConfig.getOrDefault(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false")); this.consumerTask = this.executor.submit(() -> { while (!closing) { try { - ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + ConsumerRecords records = consumer.poll(Duration.ofMillis(pollTimeoutMs)); Map offsetMap = new HashMap<>(); for (ConsumerRecord record : records) { diff --git a/driver-kop/kafka_to_kafka.yaml b/driver-kop/kafka_to_kafka.yaml new file mode 100644 index 000000000..b5e6c14b3 --- /dev/null +++ b/driver-kop/kafka_to_kafka.yaml @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Kafka producer and Pulsar consumer +driverClass: io.openmessaging.benchmark.driver.kop.KopBenchmarkDriver + +producerType: kafka +consumerType: kafka + +# Pulsar configs +pulsarConfig: + serviceUrl: pulsar://localhost:6650 + # producer configs + batchingEnabled: true + batchingMaxPublishDelayMs: 1 + batchingMaxBytes: 1048576 + blockIfQueueFull: true + pendingQueueSize: 1000 + maxPendingMessagesAcrossPartitions: 50000 + # consumer configs + maxTotalReceiverQueueSizeAcrossPartitions: 50000 + receiverQueueSize: 1000 + +# Kafka configs +kafkaConfig: | + bootstrap.servers=localhost:9092 + linger.ms=1 + batch.size=1048576 diff --git a/driver-kop/kafka_to_pulsar.yaml b/driver-kop/kafka_to_pulsar.yaml new file mode 100644 index 000000000..22d28da0e --- /dev/null +++ b/driver-kop/kafka_to_pulsar.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Kafka producer and Pulsar consumer +driverClass: io.openmessaging.benchmark.driver.kop.KopBenchmarkDriver + +producerType: kafka +consumerType: pulsar + +# Pulsar configs +pulsarConfig: + serviceUrl: pulsar://localhost:6650 + batchingMaxPublishDelayMs: 1 + batchingMaxBytes: 1048576 + +# Kafka configs +kafkaConfig: | + bootstrap.servers=localhost:9092 + linger.ms=1 + batch.size=1048576 diff --git a/driver-kop/pom.xml b/driver-kop/pom.xml new file mode 100644 index 000000000..91e014c0a --- /dev/null +++ b/driver-kop/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + .. + + + driver-kop + + + + ${project.groupId} + driver-api + ${project.version} + + + + ${project.groupId} + driver-kafka + ${project.version} + + + + ${project.groupId} + driver-pulsar + ${project.version} + + + + io.streamnative.pulsar.handlers + kafka-payload-processor + 2.9.1.5 + + + + org.testng + testng + 7.4.0 + test + + + + diff --git a/driver-kop/pulsar_to_kafka.yaml b/driver-kop/pulsar_to_kafka.yaml new file mode 100644 index 000000000..bf5c80b36 --- /dev/null +++ b/driver-kop/pulsar_to_kafka.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Kafka producer and Pulsar consumer +driverClass: io.openmessaging.benchmark.driver.kop.KopBenchmarkDriver + +producerType: pulsar +consumerType: kafka + +# Pulsar configs +pulsarConfig: + serviceUrl: pulsar://localhost:6650 + batchingMaxPublishDelayMs: 1 + batchingMaxBytes: 1048576 + +# Kafka configs +kafkaConfig: | + bootstrap.servers=localhost:9092 + linger.ms=1 + batch.size=1048576 diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java new file mode 100644 index 000000000..ec7038588 --- /dev/null +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.openmessaging.benchmark.driver.kop; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.openmessaging.benchmark.driver.BenchmarkConsumer; +import io.openmessaging.benchmark.driver.BenchmarkDriver; +import io.openmessaging.benchmark.driver.BenchmarkProducer; +import io.openmessaging.benchmark.driver.ConsumerCallback; +import io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkConsumer; +import io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkProducer; +import io.openmessaging.benchmark.driver.kop.config.ClientType; +import io.openmessaging.benchmark.driver.kop.config.Config; +import io.openmessaging.benchmark.driver.kop.config.PulsarConfig; +import io.openmessaging.benchmark.driver.pulsar.PulsarBenchmarkConsumer; +import io.openmessaging.benchmark.driver.pulsar.PulsarBenchmarkProducer; +import io.streamnative.pulsar.handlers.kop.KafkaPayloadProcessor; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerBuilder; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.util.FutureUtil; + +public class KopBenchmarkDriver implements BenchmarkDriver { + + private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + private final List producers = new CopyOnWriteArrayList<>(); + private final List consumers = new CopyOnWriteArrayList<>(); + + private Config config; + private AdminClient admin; + private Properties producerProperties; + private Properties consumerProperties; + private PulsarClient client = null; + private ProducerBuilder producerBuilder = null; + private ConsumerBuilder consumerBuilder = null; + + public static Config loadConfig(File file) throws IOException { + return mapper.readValue(file, Config.class); + } + + @Override + public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException { + config = loadConfig(configurationFile); + final Properties commonProperties = config.getKafkaProperties(); + admin = AdminClient.create(commonProperties); + + producerProperties = new Properties(); + commonProperties.forEach(producerProperties::put); + producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + + consumerProperties = new Properties(); + commonProperties.forEach(consumerProperties::put); + consumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); + + final PulsarConfig pulsarConfig = config.pulsarConfig; + if (config.producerType.equals(ClientType.PULSAR)) { + producerBuilder = getPulsarClient(pulsarConfig.serviceUrl).newProducer() + .enableBatching(pulsarConfig.batchingEnabled) + .blockIfQueueFull(pulsarConfig.blockIfQueueFull) + .batchingMaxPublishDelay(pulsarConfig.batchingMaxPublishDelayMs, TimeUnit.MILLISECONDS) + .batchingMaxBytes(pulsarConfig.batchingMaxBytes) + .maxPendingMessages(pulsarConfig.pendingQueueSize) + .maxPendingMessagesAcrossPartitions(pulsarConfig.maxPendingMessagesAcrossPartitions); + } + if (config.consumerType.equals(ClientType.PULSAR)) { + consumerBuilder = getPulsarClient(pulsarConfig.serviceUrl).newConsumer(Schema.BYTEBUFFER) + .subscriptionType(SubscriptionType.Failover) + .receiverQueueSize(pulsarConfig.receiverQueueSize) + .maxTotalReceiverQueueSizeAcrossPartitions(pulsarConfig.maxTotalReceiverQueueSizeAcrossPartitions); + } + } + + @Override + public String getTopicNamePrefix() { + return "test-topic"; + } + + @Override + public CompletableFuture createTopic(String topic, int partitions) { + // replicationFactor is meaningless in KoP + final NewTopic newTopic = new NewTopic(topic, partitions, (short) 1L); + final CompletableFuture future = new CompletableFuture<>(); + admin.createTopics(Collections.singletonList(newTopic)).all().whenComplete((result, throwable) -> { + if (throwable == null) { + future.complete(result); + } else { + future.completeExceptionally(throwable); + } + }); + return future; + } + + @Override + public CompletableFuture createProducer(String topic) { + if (config.producerType.equals(ClientType.KAFKA)) { + final BenchmarkProducer producer = + new KafkaBenchmarkProducer(new KafkaProducer<>(producerProperties), topic); + producers.add(producer); + return CompletableFuture.completedFuture(producer); + } else if (config.consumerType.equals(ClientType.PULSAR)) { + return producerBuilder.clone().topic(topic).createAsync().thenApply(PulsarBenchmarkProducer::new); + } else { + throw new IllegalArgumentException("producerType " + config.producerType + " is invalid"); + } + } + + @Override + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { + if (config.consumerType.equals(ClientType.KAFKA)) { + final Properties properties = new Properties(); + consumerProperties.forEach(properties::put); + properties.put(ConsumerConfig.GROUP_ID_CONFIG, subscriptionName); + properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + final KafkaConsumer kafkaConsumer = new KafkaConsumer<>(properties); + kafkaConsumer.subscribe(Collections.singleton(topic)); + final BenchmarkConsumer consumer = new KafkaBenchmarkConsumer( + kafkaConsumer, properties, consumerCallback, config.pollTimeoutMs); + consumers.add(consumer); + return CompletableFuture.completedFuture(consumer); + } else if (config.consumerType.equals(ClientType.PULSAR)) { + final List>> futures = new ArrayList<>(); + return client.getPartitionsForTopic(topic) + .thenCompose(partitions -> { + partitions.forEach(p -> futures.add( + createInternalPulsarConsumer(p, subscriptionName, consumerCallback))); + return FutureUtil.waitForAll(futures); + }).thenApply(__ -> new PulsarBenchmarkConsumer(futures.stream().map(CompletableFuture::join) + .collect(Collectors.toList()))); + } else { + throw new IllegalArgumentException("consumerType " + config.consumerType + " is invalid"); + } + } + + @Override + public void close() throws Exception { + for (BenchmarkProducer producer : producers) { + producer.close(); + } + for (BenchmarkConsumer consumer : consumers) { + consumer.close(); + } + admin.close(); + if (client != null) { + client.close(); + } + } + + private PulsarClient getPulsarClient(String serviceUrl) throws PulsarClientException { + if (client == null) { + client = PulsarClient.builder().serviceUrl(serviceUrl).build(); + } + return client; + } + + private CompletableFuture> createInternalPulsarConsumer( + String topic, String subscriptionName, ConsumerCallback callback) { + return consumerBuilder.clone() + .topic(topic) + .subscriptionName(subscriptionName) + .messagePayloadProcessor(new KafkaPayloadProcessor()) // support consuming Kafka format messages + .poolMessages(true) + .messageListener((c, msg) -> { + try { + callback.messageReceived(msg.getValue(), msg.getPublishTime()); + c.acknowledgeAsync(msg); + } finally { + msg.release(); + } + }) + .subscribeAsync(); + } +} diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java new file mode 100644 index 000000000..ac71aaf04 --- /dev/null +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.openmessaging.benchmark.driver.kop.config; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public enum ClientType { + @JsonProperty("kafka") KAFKA, + @JsonProperty("pulsar") PULSAR +} diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java new file mode 100644 index 000000000..2456413af --- /dev/null +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.openmessaging.benchmark.driver.kop.config; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Properties; +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.producer.ProducerConfig; + +public class Config { + + public ClientType producerType; + public ClientType consumerType; + public long pollTimeoutMs = 100; + public PulsarConfig pulsarConfig; + public String kafkaConfig; + + public Properties getKafkaProperties() { + if (StringUtils.isEmpty(kafkaConfig)) { + throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " is not set"); + } + final Properties props = new Properties(); + try { + props.load(new StringReader(kafkaConfig)); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + if (props.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG) == null) { + throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " is not set"); + } + return props; + } +} diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java new file mode 100644 index 000000000..622d60455 --- /dev/null +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.openmessaging.benchmark.driver.kop.config; + +public class PulsarConfig { + + public String serviceUrl; + + // producer configs + public boolean batchingEnabled = true; + public boolean blockIfQueueFull = true; + public int batchingMaxPublishDelayMs = 1; + public int batchingMaxBytes = 128 * 1024; + public int pendingQueueSize = 1000; + public int maxPendingMessagesAcrossPartitions = 50000; + public int batchingPartitionSwitchFrequencyByPublishDelay = 10; + + // consumer configs + public int maxTotalReceiverQueueSizeAcrossPartitions = 50000; + public int receiverQueueSize = 1000; +} diff --git a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java new file mode 100644 index 000000000..3717d3812 --- /dev/null +++ b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.openmessaging.benchmark.driver.kop; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Properties; +import io.openmessaging.benchmark.driver.kop.config.ClientType; +import io.openmessaging.benchmark.driver.kop.config.Config; +import io.openmessaging.benchmark.driver.kop.config.PulsarConfig; +import org.testng.annotations.Test; + +public class KopBenchmarkDriverTest { + + @Test + public void testLoadDefaultConfig() throws URISyntaxException, IOException { + final URL url = getClass().getClassLoader().getResource("kop_required.yaml"); + assertNotNull(url); + + final Config config = KopBenchmarkDriver.loadConfig(new File(url.toURI())); + assertEquals(config.producerType, ClientType.PULSAR); + assertEquals(config.consumerType, ClientType.KAFKA); + assertEquals(config.pollTimeoutMs, 100); + + final PulsarConfig pulsarConfig = config.pulsarConfig; + assertEquals(pulsarConfig.serviceUrl, "pulsar://localhost:6650"); + assertTrue(pulsarConfig.batchingEnabled); + assertEquals(pulsarConfig.batchingMaxPublishDelayMs, 1); + assertEquals(pulsarConfig.batchingMaxBytes, 131072); + assertTrue(pulsarConfig.blockIfQueueFull); + assertEquals(pulsarConfig.pendingQueueSize, 1000); + assertEquals(pulsarConfig.maxPendingMessagesAcrossPartitions, 50000); + assertEquals(pulsarConfig.batchingPartitionSwitchFrequencyByPublishDelay, 10); + assertEquals(pulsarConfig.maxTotalReceiverQueueSizeAcrossPartitions, 50000); + assertEquals(pulsarConfig.receiverQueueSize, 1000); + + assertEquals(config.getKafkaProperties().get("bootstrap.servers"), "localhost:9092"); + } + + @Test + public void testLoadCustomConfig() throws Exception { + final URL url = this.getClass().getClassLoader().getResource("kop_custom.yaml"); + assertNotNull(url); + + final Config config = KopBenchmarkDriver.loadConfig(new File(url.toURI())); + assertEquals(config.producerType, ClientType.KAFKA); + assertEquals(config.consumerType, ClientType.PULSAR); + assertEquals(config.pollTimeoutMs, 1000); + + final PulsarConfig pulsarConfig = config.pulsarConfig; + assertEquals(pulsarConfig.serviceUrl, "pulsar+ssl://localhost:6651"); + assertFalse(pulsarConfig.batchingEnabled); + assertEquals(pulsarConfig.batchingMaxPublishDelayMs, 10); + assertEquals(pulsarConfig.batchingMaxBytes, 1310720); + assertFalse(pulsarConfig.blockIfQueueFull); + assertEquals(pulsarConfig.pendingQueueSize, 10000); + assertEquals(pulsarConfig.maxPendingMessagesAcrossPartitions, 500000); + assertEquals(pulsarConfig.batchingPartitionSwitchFrequencyByPublishDelay, 100); + assertEquals(pulsarConfig.maxTotalReceiverQueueSizeAcrossPartitions, 500000); + assertEquals(pulsarConfig.receiverQueueSize, 10000); + + final Properties props = config.getKafkaProperties(); + assertEquals(props.size(), 3); + assertEquals(props.get("bootstrap.servers"), "localhost:9092"); + assertEquals(props.get("linger.ms"), "1"); + assertEquals(props.get("batch.size"), "1048576"); + } + + @Test + public void testLoadWrongKafkaConfig() throws Exception { + final URL url = this.getClass().getClassLoader().getResource("kop_wrong_kafka_config.yaml"); + assertNotNull(url); + + final Config config = KopBenchmarkDriver.loadConfig(new File(url.toURI())); + try { + config.getKafkaProperties(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "bootstrap.servers is not set"); + } + } +} diff --git a/driver-kop/src/test/resources/kop_custom.yaml b/driver-kop/src/test/resources/kop_custom.yaml new file mode 100644 index 000000000..cf9fd3a06 --- /dev/null +++ b/driver-kop/src/test/resources/kop_custom.yaml @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# The KoP config file whose optional items are customized with different values from default. + +name: Kafka-on-Pulsar +driverClass: io.openmessaging.benchmark.driver.kop.KopBenchmarkDriver + +producerType: kafka +consumerType: pulsar +pollTimeoutMs: 1000 + +# Pulsar configs +pulsarConfig: + serviceUrl: pulsar+ssl://localhost:6651 + # producer configs + batchingEnabled: false + batchingMaxPublishDelayMs: 10 + batchingMaxBytes: 1310720 + blockIfQueueFull: false + pendingQueueSize: 10000 + maxPendingMessagesAcrossPartitions: 500000 + # consumer configs + maxTotalReceiverQueueSizeAcrossPartitions: 500000 + receiverQueueSize: 10000 + +# Kafka configs +kafkaConfig: | + bootstrap.servers=localhost:9092 + linger.ms=1 + batch.size=1048576 diff --git a/driver-kop/src/test/resources/kop_required.yaml b/driver-kop/src/test/resources/kop_required.yaml new file mode 100644 index 000000000..2547ee587 --- /dev/null +++ b/driver-kop/src/test/resources/kop_required.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# The KoP config file with required config items. + +name: Kafka-on-Pulsar +driverClass: io.openmessaging.benchmark.driver.kop.KopBenchmarkDriver + +producerType: pulsar +consumerType: kafka + +# Pulsar configs +pulsarConfig: + serviceUrl: pulsar://localhost:6650 + +# Kafka configs +kafkaConfig: | + bootstrap.servers=localhost:9092 diff --git a/driver-kop/src/test/resources/kop_wrong_kafka_config.yaml b/driver-kop/src/test/resources/kop_wrong_kafka_config.yaml new file mode 100644 index 000000000..8baaa47a7 --- /dev/null +++ b/driver-kop/src/test/resources/kop_wrong_kafka_config.yaml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# The KoP config file whose kafka config is wrong (without required configs) + +name: Kafka-on-Pulsar +driverClass: io.openmessaging.benchmark.driver.kop.KopBenchmarkDriver + +kafkaConfig: + linger.ms=1 diff --git a/pom.xml b/pom.xml index 047c6cbea..4cd0ba4f8 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ driver-redis package docker + driver-kop From 380acf79ac5c8a52937b81cb6152deae7b7e041c Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 10 Mar 2022 16:06:59 +0800 Subject: [PATCH 007/103] [Pulsar] Fix available zone not configured and clean up the terraform script (#246) * [Pulsar] Fix available zone not configured and clean up the terraform script * Fix wrong output of zookeeper instances --- .../deploy/ssd/provision-pulsar-aws.tf | 127 +++++++++++------- driver-pulsar/deploy/ssd/terraform.tfvars | 16 +-- 2 files changed, 89 insertions(+), 54 deletions(-) diff --git a/driver-pulsar/deploy/ssd/provision-pulsar-aws.tf b/driver-pulsar/deploy/ssd/provision-pulsar-aws.tf index 7c54a89bf..34e67700b 100644 --- a/driver-pulsar/deploy/ssd/provision-pulsar-aws.tf +++ b/driver-pulsar/deploy/ssd/provision-pulsar-aws.tf @@ -1,10 +1,14 @@ -provider "aws" { - region = "${var.region}" - version = "3.50" -} - -provider "random" { - version = "3.1" +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + random = { + source = "hashicorp/random" + version = "3.1" + } + } } variable "public_key_path" { @@ -27,15 +31,13 @@ variable "key_name" { } variable "region" {} - +variable "az" {} variable "ami" {} +variable "instance_types" {} +variable "num_instances" {} -variable "instance_types" { - type = map(string) -} - -variable "num_instances" { - type = map(string) +provider "aws" { + region = var.region } # Create a VPC to launch our instances into @@ -49,26 +51,27 @@ resource "aws_vpc" "benchmark_vpc" { # Create an internet gateway to give our subnet access to the outside world resource "aws_internet_gateway" "pulsar" { - vpc_id = "${aws_vpc.benchmark_vpc.id}" + vpc_id = aws_vpc.benchmark_vpc.id } # Grant the VPC internet access on its main route table resource "aws_route" "internet_access" { - route_table_id = "${aws_vpc.benchmark_vpc.main_route_table_id}" + route_table_id = aws_vpc.benchmark_vpc.main_route_table_id destination_cidr_block = "0.0.0.0/0" - gateway_id = "${aws_internet_gateway.pulsar.id}" + gateway_id = aws_internet_gateway.pulsar.id } # Create a subnet to launch our instances into resource "aws_subnet" "benchmark_subnet" { - vpc_id = "${aws_vpc.benchmark_vpc.id}" + vpc_id = aws_vpc.benchmark_vpc.id cidr_block = "10.0.0.0/24" map_public_ip_on_launch = true + availability_zone = var.az } resource "aws_security_group" "benchmark_security_group" { name = "terraform-pulsar-${random_id.hash.hex}" - vpc_id = "${aws_vpc.benchmark_vpc.id}" + vpc_id = aws_vpc.benchmark_vpc.id # SSH access from anywhere ingress { @@ -115,16 +118,17 @@ resource "aws_security_group" "benchmark_security_group" { resource "aws_key_pair" "auth" { key_name = "${var.key_name}-${random_id.hash.hex}" - public_key = "${file(var.public_key_path)}" + public_key = file(var.public_key_path) } resource "aws_instance" "zookeeper" { - ami = "${var.ami}" - instance_type = "${var.instance_types["zookeeper"]}" - key_name = "${aws_key_pair.auth.id}" - subnet_id = "${aws_subnet.benchmark_subnet.id}" - vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"] - count = "${var.num_instances["zookeeper"]}" + ami = var.ami + instance_type = var.instance_types["zookeeper"] + key_name = aws_key_pair.auth.id + subnet_id = aws_subnet.benchmark_subnet.id + vpc_security_group_ids = [ + aws_security_group.benchmark_security_group.id] + count = var.num_instances["zookeeper"] tags = { Name = "zk-${count.index}" @@ -132,12 +136,13 @@ resource "aws_instance" "zookeeper" { } resource "aws_instance" "pulsar" { - ami = "${var.ami}" - instance_type = "${var.instance_types["pulsar"]}" - key_name = "${aws_key_pair.auth.id}" - subnet_id = "${aws_subnet.benchmark_subnet.id}" - vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"] - count = "${var.num_instances["pulsar"]}" + ami = var.ami + instance_type = var.instance_types["pulsar"] + key_name = aws_key_pair.auth.id + subnet_id = aws_subnet.benchmark_subnet.id + vpc_security_group_ids = [ + aws_security_group.benchmark_security_group.id] + count = var.num_instances["pulsar"] tags = { Name = "pulsar-${count.index}" @@ -145,12 +150,13 @@ resource "aws_instance" "pulsar" { } resource "aws_instance" "client" { - ami = "${var.ami}" - instance_type = "${var.instance_types["client"]}" - key_name = "${aws_key_pair.auth.id}" - subnet_id = "${aws_subnet.benchmark_subnet.id}" - vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"] - count = "${var.num_instances["client"]}" + ami = var.ami + instance_type = var.instance_types["client"] + key_name = aws_key_pair.auth.id + subnet_id = aws_subnet.benchmark_subnet.id + vpc_security_group_ids = [ + aws_security_group.benchmark_security_group.id] + count = var.num_instances["client"] tags = { Name = "pulsar-client-${count.index}" @@ -158,22 +164,51 @@ resource "aws_instance" "client" { } resource "aws_instance" "prometheus" { - ami = "${var.ami}" - instance_type = "${var.instance_types["prometheus"]}" - key_name = "${aws_key_pair.auth.id}" - subnet_id = "${aws_subnet.benchmark_subnet.id}" - vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"] - count = "${var.num_instances["prometheus"]}" + ami = var.ami + instance_type = var.instance_types["prometheus"] + key_name = aws_key_pair.auth.id + subnet_id = aws_subnet.benchmark_subnet.id + vpc_security_group_ids = [ + aws_security_group.benchmark_security_group.id] + count = var.num_instances["prometheus"] tags = { Name = "prometheus-${count.index}" } } +output "zookeeper" { + value = { + for instance in aws_instance.zookeeper : + instance.public_ip => instance.private_ip + } +} + +output "pulsar" { + value = { + for instance in aws_instance.pulsar : + instance.public_ip => instance.private_ip + } +} + +output "client" { + value = { + for instance in aws_instance.client : + instance.public_ip => instance.private_ip + } +} + +output "prometheus" { + value = { + for instance in aws_instance.prometheus : + instance.public_ip => instance.private_ip + } +} + output "client_ssh_host" { - value = "${aws_instance.client.0.public_ip}" + value = aws_instance.client.0.public_ip } output "prometheus_host" { - value = "${aws_instance.prometheus.0.public_ip}" + value = aws_instance.prometheus.0.public_ip } diff --git a/driver-pulsar/deploy/ssd/terraform.tfvars b/driver-pulsar/deploy/ssd/terraform.tfvars index 4de684d91..62b012179 100644 --- a/driver-pulsar/deploy/ssd/terraform.tfvars +++ b/driver-pulsar/deploy/ssd/terraform.tfvars @@ -4,15 +4,15 @@ az = "us-west-2a" ami = "ami-9fa343e7" // RHEL-7.4 instance_types = { - "pulsar" = "i3en.6xlarge" - "zookeeper" = "t2.small" - "client" = "m5n.8xlarge" - "prometheus" = "t2.large" + "pulsar" = "i3en.6xlarge" + "zookeeper" = "t2.small" + "client" = "m5n.8xlarge" + "prometheus" = "t2.large" } num_instances = { - "client" = 4 - "pulsar" = 3 - "zookeeper" = 3 - "prometheus" = 1 + "client" = 4 + "pulsar" = 3 + "zookeeper" = 3 + "prometheus" = 1 } From 89dce6d61c4444fa993ce36098e50ed5e124cb4a Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 10 Mar 2022 16:11:08 +0800 Subject: [PATCH 008/103] Support deploying protocol handlers for Pulsar (#251) * Support configuring protocol handlers and many other options * Fix deployment failure for RedHat 8 * Fix protocol handlers not loaded * Adjust the extra_vars.yaml * Use the original name of deloy.yaml * Add document for skip_journal * Update README * Support configuring max heap/direct memory --- driver-pulsar/README.md | 49 ++++++ .../{deploy_with_stats.yaml => deploy.yaml} | 161 ++++++++++++++---- driver-pulsar/deploy/ssd/extra_vars.yaml | 62 +++++++ driver-pulsar/deploy/ssd/templates/kop.conf | 24 +++ driver-pulsar/deploy/ssd/templates/mop.conf | 20 +++ 5 files changed, 279 insertions(+), 37 deletions(-) rename driver-pulsar/deploy/ssd/{deploy_with_stats.yaml => deploy.yaml} (71%) create mode 100644 driver-pulsar/deploy/ssd/extra_vars.yaml create mode 100644 driver-pulsar/deploy/ssd/templates/kop.conf create mode 100644 driver-pulsar/deploy/ssd/templates/mop.conf diff --git a/driver-pulsar/README.md b/driver-pulsar/README.md index c01949d89..48d2bcd78 100644 --- a/driver-pulsar/README.md +++ b/driver-pulsar/README.md @@ -1,3 +1,52 @@ # Apache Pulsar benchmarks For instructions on running the OpenMessaging benchmarks for Pulsar, see the [official documentation](http://openmessaging.cloud/docs/benchmarks/pulsar/). + +## Supplement to the official documentation + +Before you run `ansible-playbook` with `terraform-inventory`, you must set the environment variable `TF_STATE`. i.e. the completed command should be: + +```bash +TF_STATE=. ansible-playbook \ + --user ec2-user \ + --inventory `which terraform-inventory` \ + deploy.yaml +``` + +### Ansible variable files + +The Ansible deployment script supports flexible configuration with a variable file, which is specified by `-e` option like: + +```bash +TF_STATE=. ansible-playbook \ + --user ec2-user \ + --inventory `which terraform-inventory` \ + -e @extra_vars.yaml \ + deploy.yaml +``` + +For example, if you changed the AWS instance type, the two SSD device paths might not be `/dev/nvme1n1` and `/dev/nvme2n1`. In this case, you can configure them like + +```yaml +disk_dev: + - /path/to/disk1 + - /path/to/disk2 +``` + +See more explanations in [the example variable file](./deploy/ssd/extra_vars.yaml). + +### Enable protocol handlers + +With the Ansible variable file, you can enable multiple protocol handlers in `protocol_handlers` variable. For example, given following configurations: + +```yaml +protocol_handlers: + - protocol: kafka + conf: kop.conf + url: https://github.com/streamnative/kop/releases/download/v2.9.2.5/pulsar-protocol-handler-kafka-2.9.2.5.nar + - protocol: mqtt + conf: mop.conf + url: https://github.com/streamnative/mop/releases/download/v2.9.2.5/pulsar-protocol-handler-mqtt-2.9.2.5.nar +``` + +It will download KoP and MoP from the given URLs. Then, the configuration templates will be formatted and appended to the `broker.conf`. The `conf` field is the name of the configuration template, which must be put under `templates` directory. diff --git a/driver-pulsar/deploy/ssd/deploy_with_stats.yaml b/driver-pulsar/deploy/ssd/deploy.yaml similarity index 71% rename from driver-pulsar/deploy/ssd/deploy_with_stats.yaml rename to driver-pulsar/deploy/ssd/deploy.yaml index 34726d9c2..408dddd41 100644 --- a/driver-pulsar/deploy/ssd/deploy_with_stats.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -17,6 +17,37 @@ # under the License. # +- name: Initialize some default values + hosts: all + connection: ssh + tasks: + - set_fact: + private_ip: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" + - set_fact: + zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" + serviceUrl: "pulsar://{{ hostvars[groups['pulsar'][0]].private_ip }}:6650/" + httpUrl: "http://{{ hostvars[groups['pulsar'][0]].private_ip }}:8080/" + - set_fact: + pulsar_version: "{{ pulsar_version | default('2.9.1') }}" + node_exporter_version: "{{ node_exporter_version | default('1.2.2') }}" + prometheus_version: "{{ prometheus_version | default('2.31.1') }}" + disk_dev: "{{ disk_dev | default(['/dev/nvme1n1', '/dev/nvme2n1']) }}" + - set_fact: + pulsar_binary: + src: "https://downloads.apache.org/pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz" + remote: yes + when: pulsar_binary is not defined + - set_fact: + node_exporter_binary: + src: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz" + remote: yes + when: node_exporter_binary is not defined + - set_fact: + prometheus_binary: + src: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz" + remote: yes + when: prometheus_binary is not defined + - name: Format and mount disks for Pulsar/BookKeeper hosts hosts: pulsar connection: ssh @@ -27,8 +58,8 @@ fstype: xfs dev: '{{ item }}' with_items: - - '/dev/nvme1n1' - - '/dev/nvme2n1' + - "{{ disk_dev[0] }}" + - "{{ disk_dev[1] }}" - name: Mount disks mount: path: "{{ item.path }}" @@ -37,31 +68,28 @@ opts: defaults,noatime,nodiscard state: mounted with_items: - - { path: "/mnt/journal", src: "/dev/nvme1n1" } - - { path: "/mnt/storage", src: "/dev/nvme2n1" } + - { path: "/mnt/journal", src: "{{ disk_dev[0] }}" } + - { path: "/mnt/storage", src: "{{ disk_dev[1] }}" } - name: Install Node exporter on Brokers to collect system metrics hosts: pulsar become: true tasks: - - set_fact: - nodeExporterVersion: "1.2.2" - name: Add user node_exporter user: name: node_exporter shell: /bin/false system: true create_home: no - - name: Download and extract unarchive: - src: "https://github.com/prometheus/node_exporter/releases/download/v{{ nodeExporterVersion }}/node_exporter-{{ nodeExporterVersion }}.linux-amd64.tar.gz" + src: "{{ node_exporter_binary.src }}" dest: /tmp - remote_src: yes + remote_src: "{{ node_exporter_binary.remote }}" - name: Copy bin node_exporter to /usr/local/bin copy: - src: "/tmp/node_exporter-{{ nodeExporterVersion }}.linux-amd64/node_exporter" + src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter" remote_src: yes dest: /usr/local/bin/ owner: node_exporter @@ -102,38 +130,34 @@ timeout: 5 - name: Pulsar installation - hosts: all + hosts: pulsar:client:zookeeper connection: ssh become: true tasks: - name: Set performance profile command: tuned-adm profile latency-performance - name: Install RPM packages - yum: pkg={{ item }} state=latest - with_items: + yum: + state: latest + pkg: - wget - java-11-openjdk - java-11-openjdk-devel - sysstat - vim - chrony - - set_fact: - zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" - serviceUrl: "pulsar://{{ hostvars[groups['pulsar'][0]].private_ip }}:6650/" - httpUrl: "http://{{ hostvars[groups['pulsar'][0]].private_ip }}:8080/" - pulsarVersion: "2.9.1" + when: ansible_facts['distribution'] == 'RedHat' - file: path=/opt/pulsar state=absent - file: path=/opt/pulsar state=directory - name: Download Pulsar binary package unarchive: - src: "https://downloads.apache.org/pulsar/pulsar-{{ pulsarVersion }}/apache-pulsar-{{ pulsarVersion }}-bin.tar.gz" - remote_src: yes + src: "{{ pulsar_binary.src }}" + remote_src: "{{ pulsar_binary.remote }}" dest: /opt/pulsar extra_opts: ["--strip-components=1"] - set_fact: - private_ip: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" - max_heap_memory: "16g" - max_direct_memory: "48g" + max_heap_memory: "{{ pulsar_max_heap_memory | default('16g') }}" + max_direct_memory: "{{ pulsar_max_direct_memory | default('48g') }}" - template: src: "templates/pulsar_env.sh" dest: "/opt/pulsar/conf/pulsar_env.sh" @@ -142,6 +166,30 @@ echo 'LANG=en_US.utf-8 LC_ALL=en_US.utf-8' > /etc/environment +- name: Protocol Handlers installation + hosts: pulsar + connection: ssh + become: true + tasks: + - file: path=/opt/pulsar/protocols state=absent + - file: path=/opt/pulsar/protocols state=directory + - name: Upload local protocol handlers + copy: + src: "{{ item.local_path }}" + dest: /opt/pulsar/protocols + loop: "{{ protocol_handlers }}" + when: + - protocol_handlers is defined + - item.local_path is defined + - name: Download protocol handlers + get_url: + url: "{{ item.url }}" + dest: /opt/pulsar/protocols + loop: "{{ protocol_handlers }}" + when: + - protocol_handlers is defined + - item.local_path is not defined + - name: ZooKeeper setup hosts: zookeeper connection: ssh @@ -149,8 +197,8 @@ tasks: - set_fact: zid: "{{ groups['zookeeper'].index(inventory_hostname) }}" - max_heap_memory: "512m" - max_direct_memory: "512m" + max_heap_memory: "{{ zookeeper_max_heap_memory | default('512m') }}" + max_direct_memory: "{{ zookeeper_max_direct_memory | default('512m') }}" - file: path: "/opt/pulsar/{{ item }}" state: directory @@ -186,9 +234,9 @@ hosts: pulsar connection: ssh become: true - vars: - skip_journal: false tasks: + - set_fact: + skip_journal: "{{ skip_journal | default(false) }}" - name: Apply BookKeeper configuration file [Bypass Journal] template: src: "templates/bookkeeper-skip-journal.conf" @@ -220,6 +268,32 @@ - template: src: "templates/pulsar.service" dest: "/etc/systemd/system/pulsar.service" + - template: + src: "templates/{{ item.conf }}" + dest: "/opt/pulsar/conf/{{ item.conf }}" + loop: "{{ protocol_handlers }}" + when: protocol_handlers is defined + - set_fact: + protocols: "{{ protocols | default([]) + [ item.protocol ]}}" + loop: "{{ protocol_handlers }}" + when: protocol_handlers is defined + - name: Enable protocol handlers + lineinfile: + path: /opt/pulsar/conf/broker.conf + line: "messagingProtocols={{ protocols | join(',') }}" + when: protocols is defined + - name: Read configurations of all protocol handlers + shell: | + grep -v "^#" "{{ '/opt/pulsar/conf/' + item.conf }}" + loop: "{{ protocol_handlers }}" + when: protocol_handlers is defined + register: conf_files + - name: Update broker.conf for protocol handlers + lineinfile: + path: /opt/pulsar/conf/broker.conf + line: "{{ item.stdout_lines | join('\n') }}" + loop: "{{ conf_files.results }}" + when: conf_files is defined - systemd: state: restarted daemon_reload: yes @@ -343,28 +417,41 @@ tasks: - name: Add Extras Repo shell: yum-config-manager --enable rhui-REGION-rhel-server-extras - - name: Install RPM packages - yum: pkg={{ item }} state=latest - with_items: - - docker - - set_fact: - prometheusVersion: "2.31.1" + when: + - ansible_facts['distribution'] == 'RedHat' + - ansible_facts['distribution_major_version'] | int <= 7 + - name: Docker repo + yum_repository: + name: docker + description: repo for docker + baseurl: "https://download.docker.com/linux/centos/{{ ansible_facts['distribution_major_version'] }}/x86_64/stable/" + gpgcheck: no + when: ansible_facts['distribution'] == 'RedHat' + - name: Installing docker + yum: + state: latest + pkg: ['docker-ce'] + - name: Start docker + service: + name: docker + state: started + enabled: yes - file: path=/opt/prometheus state=absent - file: path=/opt/prometheus state=directory - name: Download Prometheus Binary Package unarchive: - src: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheusVersion }}/prometheus-{{ prometheusVersion }}.linux-amd64.tar.gz" - remote_src: yes + src: "{{ prometheus_binary.src }}" + remote_src: "{{ prometheus_binary.remote }}" dest: /opt/prometheus extra_opts: ["--strip-components=1"] - - set_fact: - private_ip: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" - name: Prometheus setup hosts: prometheus connection: ssh become: true tasks: + - file: path=/opt/pulsar state=absent + - file: path=/opt/pulsar state=directory - file: path: "/opt/prometheus/{{ item }}" state: directory diff --git a/driver-pulsar/deploy/ssd/extra_vars.yaml b/driver-pulsar/deploy/ssd/extra_vars.yaml new file mode 100644 index 000000000..684b170c0 --- /dev/null +++ b/driver-pulsar/deploy/ssd/extra_vars.yaml @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +pulsar_version: 2.9.1 +node_exporter_version: 1.2.2 +prometheus_version: 2.31.1 + +pulsar_max_heap_memory: 16g +pulsar_max_direct_memory: 48g + +zookeeper_max_heap_memory: 512m +zookeeper_max_direct_memory: 512m + +# Set it with true if you want BookKeeper to skip the journal writes. +skip_journal: false + +# By default, ansible downloads binaries from network. The download URL is determined automatically by the version. +# To specific another URL to download, you should configure `remote` with `yes` and `src` with the new URL. +# To upload local binaries, you should configure `remote` with `no` and `src` with the local path. +#pulsar_binary: +# src: ./apache-pulsar-2.9.1-bin.tar.gz +# remote: no +# +#node_exporter_binary: +# src: ./node_exporter-1.2.2.linux-amd64.tar.gz +# remote: no +# +#prometheus_binary: +# src: ./prometheus-2.31.1.linux-amd64.tar.gz +# remote: no + +# The two disk device paths as BookKeeper journal and storage disks +disk_dev: + - /dev/nvme1n1 + - /dev/nvme2n1 + +# Each item represents a protocol handler. If `local_path` is defined, the NAR file will be uploaded from local path instead of downloading from `url`. +protocol_handlers: + - protocol: kafka + conf: kop.conf + url: https://github.com/streamnative/kop/releases/download/v2.9.2.5/pulsar-protocol-handler-kafka-2.9.2.5.nar + #local_path: ./pulsar-protocol-handler-kafka-2.9.2.5.nar + - protocol: mqtt + conf: mop.conf + url: https://github.com/streamnative/mop/releases/download/v2.9.2.5/pulsar-protocol-handler-mqtt-2.9.2.5.nar + #local_path: ./pulsar-protocol-handler-mqtt-2.9.2.5.nar diff --git a/driver-pulsar/deploy/ssd/templates/kop.conf b/driver-pulsar/deploy/ssd/templates/kop.conf new file mode 100644 index 000000000..6bed09cec --- /dev/null +++ b/driver-pulsar/deploy/ssd/templates/kop.conf @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +kafkaListeners=PLAINTEXT://{{ hostvars[inventory_hostname].private_ip }}:9092 +kafkaAdvertisedListeners=PLAINTEXT://{{ hostvars[inventory_hostname].private_ip }}:9092 +allowAutoTopicCreationType=partitioned +brokerEntryMetadataInterceptors=org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor +brokerDeleteInactiveTopicsEnabled=false diff --git a/driver-pulsar/deploy/ssd/templates/mop.conf b/driver-pulsar/deploy/ssd/templates/mop.conf new file mode 100644 index 000000000..89002b63b --- /dev/null +++ b/driver-pulsar/deploy/ssd/templates/mop.conf @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +mqttListeners=mqtt://{{ hostvars[inventory_hostname].private_ip }}:1883 From 1701fca656bfedf7f6eb6b30e32b6c0e41e81a91 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 29 Mar 2022 23:03:24 +0800 Subject: [PATCH 009/103] [Pulsar] Add ansible playbook to restart broker (#257) --- driver-pulsar/README.md | 12 +++ driver-pulsar/deploy/ssd/restart_broker.yaml | 85 ++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 driver-pulsar/deploy/ssd/restart_broker.yaml diff --git a/driver-pulsar/README.md b/driver-pulsar/README.md index 48d2bcd78..74446bd07 100644 --- a/driver-pulsar/README.md +++ b/driver-pulsar/README.md @@ -50,3 +50,15 @@ protocol_handlers: ``` It will download KoP and MoP from the given URLs. Then, the configuration templates will be formatted and appended to the `broker.conf`. The `conf` field is the name of the configuration template, which must be put under `templates` directory. + +### Restart the brokers with new configurations + +You can change the configuration files and then restart the cluster by executing the following command. + +```bash +TF_STATE=. ansible-playbook \ + --user ec2-user \ + --inventory `which terraform-inventory` \ + -e @extra_vars.yaml \ + restart_broker.yaml +``` diff --git a/driver-pulsar/deploy/ssd/restart_broker.yaml b/driver-pulsar/deploy/ssd/restart_broker.yaml new file mode 100644 index 000000000..ff43cc233 --- /dev/null +++ b/driver-pulsar/deploy/ssd/restart_broker.yaml @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +- name: Initialize variables + hosts: all + connection: ssh + tasks: + - set_fact: + private_ip: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" + zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" + +- name: Restart brokers with new configurations + hosts: pulsar + become: true + tasks: + - set_fact: + max_heap_memory: "{{ pulsar_max_heap_memory | default('16g') }}" + max_direct_memory: "{{ pulsar_max_direct_memory | default('48g') }}" + - systemd: + state: stopped + name: pulsar + - name: Set up pulsar.service + template: + src: templates/pulsar.service + dest: /etc/systemd/system/pulsar.service + - name: Set up pulsar_env.sh + template: + src: templates/pulsar_env.sh + dest: /opt/pulsar/conf/pulsar_env.sh + - name: Set up broker.conf + template: + src: templates/broker.conf + dest: /opt/pulsar/conf/broker.conf + - name: Set up configurations of protocol handlers + template: + src: "templates/{{ item.conf }}" + dest: "/opt/pulsar/conf/{{ item.conf }}" + loop: "{{ protocol_handlers }}" + when: protocol_handlers is defined + - set_fact: + protocols: "{{ protocols | default([]) + [ item.protocol ] }}" + loop: "{{ protocol_handlers }}" + when: protocol_handlers is defined + - name: Enable protocol handlers + lineinfile: + path: /opt/pulsar/conf/broker.conf + line: "messagingProtocols={{ protocols | join(',') }}" + - name: Read configurations of all protocol handlers + shell: | + grep -v "^#" "{{ '/opt/pulsar/conf/' + item.conf }}" + loop: "{{ protocol_handlers }}" + when: protocol_handlers is defined + register: conf_files + - name: Read configurations of all protocol handlers + shell: | + grep -v "^#" "{{ '/opt/pulsar/conf/' + item.conf }}" + loop: "{{ protocol_handlers }}" + when: protocol_handlers is defined + register: conf_files + - name: Update broker.conf for protocol handlers + lineinfile: + path: /opt/pulsar/conf/broker.conf + line: "{{ item.stdout_lines | join('\n') }}" + loop: "{{ conf_files.results }}" + when: conf_files is defined + - systemd: + state: started + daemon_reload: yes + name: pulsar From 800e1afd5863baa4c9e932b94a2f2e69fe480a23 Mon Sep 17 00:00:00 2001 From: Andrey Koltsov Date: Thu, 7 Apr 2022 04:59:04 +0300 Subject: [PATCH 010/103] [Pravega Driver] [Draft] Update dependencies for Pravega 0.12 (#259) * Update gRPC version for Pravega 0.12. Signed-off-by: Andrey Koltsov * Add comment @ pom.xml Signed-off-by: AndreyKoltsov1997 --- driver-pravega/pom.xml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/driver-pravega/pom.xml b/driver-pravega/pom.xml index 1afdbc2cf..eea42d368 100644 --- a/driver-pravega/pom.xml +++ b/driver-pravega/pom.xml @@ -30,9 +30,9 @@ driver-pravega - - - 0.10.1 + + + 0.12.0 @@ -68,7 +68,14 @@ io.grpc grpc-all - 1.36.0 + + 1.39.0 + + + + io.netty + netty-tcnative-boringssl-static + 2.0.39.Final From 0d02d56332c425ec557715b9327c597b5094a7ce Mon Sep 17 00:00:00 2001 From: cnScarb Date: Thu, 7 Apr 2022 10:01:23 +0800 Subject: [PATCH 011/103] Fix RabbitMQ docs (#255) (#256) * update git repository address * Cannot create topic with partitions in RabbitMQ, so use 1 partition for rabbitmq --- driver-rabbitmq/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/driver-rabbitmq/README.md b/driver-rabbitmq/README.md index b1bbc0ca9..16b9b1400 100644 --- a/driver-rabbitmq/README.md +++ b/driver-rabbitmq/README.md @@ -12,7 +12,7 @@ This folder houses all of the assets necessary to run benchmarks for [RabbitMQ]( In order to create the local artifacts necessary to run the RabbitMQ benchmarks in AWS, you'll need to have [Maven](https://maven.apache.org/install.html) installed. Once Maven's installed, you can create the necessary artifacts with a single Maven command: ```bash -$ git clone https://github.com/streamlio/messaging-benchmark +$ git clone https://github.com/openmessaging/benchmark.git % cd messaging-benchmark $ mvn install ``` @@ -108,5 +108,5 @@ $ sudo bin/benchmark --drivers driver-rabbitmq/rabbitmq.yaml workloads/*.yaml You can also run specific workloads in the `workloads` folder. Here's an example: ```bash -$ sudo bin/benchmark --drivers driver-rabbotmq/rabbitmq.yaml workloads/1-topic-16-partitions-1kb.yaml +$ sudo bin/benchmark --drivers driver-rabbotmq/rabbitmq.yaml workloads/1-topic-1-partitions-1kb.yaml ``` \ No newline at end of file From 9a8254dbc8e5a1daba752c48032fb22c0843ce17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:03:03 +0800 Subject: [PATCH 012/103] Bump jackson-databind from 2.9.10.8 to 2.12.6.1 (#261) Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.9.10.8 to 2.12.6.1. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4cd0ba4f8..4a9bcfb5c 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ com.fasterxml.jackson.core jackson-databind - 2.9.10.8 + 2.12.6.1 From 64cc01df8eff4042a4502f0390358f1246e1e146 Mon Sep 17 00:00:00 2001 From: novosibman Date: Thu, 7 Apr 2022 09:06:21 +0700 Subject: [PATCH 013/103] Test warmup option (#260) --- .../main/java/io/openmessaging/benchmark/Workload.java | 2 ++ .../io/openmessaging/benchmark/WorkloadGenerator.java | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java index 32f770e87..844360e65 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java @@ -57,4 +57,6 @@ public class Workload { public long consumerBacklogSizeGB = 0; public int testDurationMinutes; + + public int warmupDurationMinutes = 1; } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 592bfa973..023e1a362 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -122,12 +122,12 @@ public TestResult run() throws Exception { producerWorkAssignment.payloadData.add(payloadReader.load(workload.payloadFile)); } - - log.info("----- Starting warm-up traffic ------"); - worker.startLoad(producerWorkAssignment); - printAndCollectStats(1, TimeUnit.MINUTES); + if (workload.warmupDurationMinutes > 0) { + log.info("----- Starting warm-up traffic ({}m) ------", workload.warmupDurationMinutes); + printAndCollectStats(workload.warmupDurationMinutes, TimeUnit.MINUTES); + } if (workload.consumerBacklogSizeGB > 0) { executor.execute(() -> { @@ -140,7 +140,7 @@ public TestResult run() throws Exception { } worker.resetStats(); - log.info("----- Starting benchmark traffic ------"); + log.info("----- Starting benchmark traffic ({}m)------", workload.testDurationMinutes); TestResult result = printAndCollectStats(workload.testDurationMinutes, TimeUnit.MINUTES); runCompleted = true; From 64eff253ddb5be19a8333a7cc894c5ca37a0cefe Mon Sep 17 00:00:00 2001 From: Francesco Nigro Date: Thu, 7 Apr 2022 04:23:30 +0200 Subject: [PATCH 014/103] Fixes #247 Fix Coordinated Omission (#248) --- .../openmessaging/benchmark/TestResult.java | 20 +++++ .../benchmark/WorkloadGenerator.java | 44 ++++++++++- .../benchmark/utils/UniformRateLimiter.java | 79 +++++++++++++++++++ .../worker/DistributedWorkersEnsemble.java | 12 +++ .../benchmark/worker/LocalWorker.java | 33 ++++++-- .../benchmark/worker/WorkerHandler.java | 12 +++ .../worker/commands/CumulativeLatencies.java | 4 + .../worker/commands/PeriodStats.java | 5 ++ bin/create_charts.py | 8 ++ 9 files changed, 206 insertions(+), 11 deletions(-) create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java index 0a52b21c0..57c140094 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java @@ -45,6 +45,15 @@ public class TestResult { public List publishLatency9999pct = new ArrayList<>(); public List publishLatencyMax = new ArrayList<>(); + public List publishDelayLatencyAvg = new ArrayList<>(); + public List publishDelayLatency50pct = new ArrayList<>(); + public List publishDelayLatency75pct = new ArrayList<>(); + public List publishDelayLatency95pct = new ArrayList<>(); + public List publishDelayLatency99pct = new ArrayList<>(); + public List publishDelayLatency999pct = new ArrayList<>(); + public List publishDelayLatency9999pct = new ArrayList<>(); + public List publishDelayLatencyMax = new ArrayList<>(); + public double aggregatedPublishLatencyAvg; public double aggregatedPublishLatency50pct; public double aggregatedPublishLatency75pct; @@ -54,8 +63,19 @@ public class TestResult { public double aggregatedPublishLatency9999pct; public double aggregatedPublishLatencyMax; + public double aggregatedPublishDelayLatencyAvg; + public long aggregatedPublishDelayLatency50pct; + public long aggregatedPublishDelayLatency75pct; + public long aggregatedPublishDelayLatency95pct; + public long aggregatedPublishDelayLatency99pct; + public long aggregatedPublishDelayLatency999pct; + public long aggregatedPublishDelayLatency9999pct; + public long aggregatedPublishDelayLatencyMax; + public Map aggregatedPublishLatencyQuantiles = new TreeMap<>(); + public Map aggregatedPublishDelayLatencyQuantiles = new TreeMap<>(); + // End to end latencies (from producer to consumer) // Latencies are expressed in milliseconds (without decimals) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 023e1a362..780175cac 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -416,7 +416,7 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw - stats.totalMessagesReceived; log.info( - "Pub rate {} msg/s / {} MB/s | Cons rate {} msg/s / {} MB/s | Backlog: {} K | Pub Latency (ms) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {}", + "Pub rate {} msg/s / {} MB/s | Cons rate {} msg/s / {} MB/s | Backlog: {} K | Pub Latency (ms) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {} | Pub Delay Latency (us) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {}", rateFormat.format(publishRate), throughputFormat.format(publishThroughput), rateFormat.format(consumeRate), throughputFormat.format(consumeThroughput), dec.format(currentBacklog / 1000.0), // @@ -424,7 +424,12 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(50))), dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(99))), dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(99.9))), - throughputFormat.format(microsToMillis(stats.publishLatency.getMaxValue()))); + throughputFormat.format(microsToMillis(stats.publishLatency.getMaxValue())), + dec.format(stats.publishDelayLatency.getMean()), + dec.format(stats.publishDelayLatency.getValueAtPercentile(50)), + dec.format(stats.publishDelayLatency.getValueAtPercentile(99)), + dec.format(stats.publishDelayLatency.getValueAtPercentile(99.9)), + throughputFormat.format(stats.publishDelayLatency.getMaxValue())); result.publishRate.add(publishRate); result.consumeRate.add(consumeRate); @@ -438,6 +443,16 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw result.publishLatency9999pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99.99))); result.publishLatencyMax.add(microsToMillis(stats.publishLatency.getMaxValue())); + result.publishDelayLatencyAvg.add(stats.publishDelayLatency.getMean()); + result.publishDelayLatency50pct.add(stats.publishDelayLatency.getValueAtPercentile(50)); + result.publishDelayLatency75pct.add(stats.publishDelayLatency.getValueAtPercentile(75)); + result.publishDelayLatency95pct.add(stats.publishDelayLatency.getValueAtPercentile(95)); + result.publishDelayLatency99pct.add(stats.publishDelayLatency.getValueAtPercentile(99)); + result.publishDelayLatency999pct.add(stats.publishDelayLatency.getValueAtPercentile(99.9)); + result.publishDelayLatency9999pct.add(stats.publishDelayLatency.getValueAtPercentile(99.99)); + result.publishDelayLatencyMax.add(stats.publishDelayLatency.getMaxValue()); + + result.endToEndLatencyAvg.add(microsToMillis(stats.endToEndLatency.getMean())); result.endToEndLatency50pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(50))); result.endToEndLatency75pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(75))); @@ -450,14 +465,21 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw if (now >= testEndTime && !needToWaitForBacklogDraining) { CumulativeLatencies agg = worker.getCumulativeLatencies(); log.info( - "----- Aggregated Pub Latency (ms) avg: {} - 50%: {} - 95%: {} - 99%: {} - 99.9%: {} - 99.99%: {} - Max: {}", + "----- Aggregated Pub Latency (ms) avg: {} - 50%: {} - 95%: {} - 99%: {} - 99.9%: {} - 99.99%: {} - Max: {} | Pub Delay (us) avg: {} - 50%: {} - 95%: {} - 99%: {} - 99.9%: {} - 99.99%: {} - Max: {}", dec.format(agg.publishLatency.getMean() / 1000.0), dec.format(agg.publishLatency.getValueAtPercentile(50) / 1000.0), dec.format(agg.publishLatency.getValueAtPercentile(95) / 1000.0), dec.format(agg.publishLatency.getValueAtPercentile(99) / 1000.0), dec.format(agg.publishLatency.getValueAtPercentile(99.9) / 1000.0), dec.format(agg.publishLatency.getValueAtPercentile(99.99) / 1000.0), - throughputFormat.format(agg.publishLatency.getMaxValue() / 1000.0)); + throughputFormat.format(agg.publishLatency.getMaxValue() / 1000.0), + dec.format(agg.publishDelayLatency.getMean()), + dec.format(agg.publishDelayLatency.getValueAtPercentile(50)), + dec.format(agg.publishDelayLatency.getValueAtPercentile(95)), + dec.format(agg.publishDelayLatency.getValueAtPercentile(99)), + dec.format(agg.publishDelayLatency.getValueAtPercentile(99.9)), + dec.format(agg.publishDelayLatency.getValueAtPercentile(99.99)), + throughputFormat.format(agg.publishDelayLatency.getMaxValue())); result.aggregatedPublishLatencyAvg = agg.publishLatency.getMean() / 1000.0; result.aggregatedPublishLatency50pct = agg.publishLatency.getValueAtPercentile(50) / 1000.0; @@ -468,6 +490,15 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw result.aggregatedPublishLatency9999pct = agg.publishLatency.getValueAtPercentile(99.99) / 1000.0; result.aggregatedPublishLatencyMax = agg.publishLatency.getMaxValue() / 1000.0; + result.aggregatedPublishDelayLatencyAvg = agg.publishDelayLatency.getMean(); + result.aggregatedPublishDelayLatency50pct = agg.publishDelayLatency.getValueAtPercentile(50); + result.aggregatedPublishDelayLatency75pct = agg.publishDelayLatency.getValueAtPercentile(75); + result.aggregatedPublishDelayLatency95pct = agg.publishDelayLatency.getValueAtPercentile(95); + result.aggregatedPublishDelayLatency99pct = agg.publishDelayLatency.getValueAtPercentile(99); + result.aggregatedPublishDelayLatency999pct = agg.publishDelayLatency.getValueAtPercentile(99.9); + result.aggregatedPublishDelayLatency9999pct = agg.publishDelayLatency.getValueAtPercentile(99.99); + result.aggregatedPublishDelayLatencyMax = agg.publishDelayLatency.getMaxValue(); + result.aggregatedEndToEndLatencyAvg = agg.endToEndLatency.getMean() / 1000.0; result.aggregatedEndToEndLatency50pct = agg.endToEndLatency.getValueAtPercentile(50) / 1000.0; result.aggregatedEndToEndLatency75pct = agg.endToEndLatency.getValueAtPercentile(75) / 1000.0; @@ -482,6 +513,11 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw value.getValueIteratedTo() / 1000.0); }); + agg.publishDelayLatency.percentiles(100).forEach(value -> { + result.aggregatedPublishDelayLatencyQuantiles.put(value.getPercentile(), + value.getValueIteratedTo()); + }); + agg.endToEndLatency.percentiles(100).forEach(value -> { result.aggregatedEndToEndLatencyQuantiles.put(value.getPercentile(), microsToMillis(value.getValueIteratedTo())); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java new file mode 100644 index 000000000..2c31cb1c0 --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.openmessaging.benchmark.utils; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.locks.LockSupport; + +/** + * Provides a next operation time for rate limited operation streams.
+ * The rate limiter is thread safe and can be shared by all threads. + */ +public final class UniformRateLimiter { + + private static final AtomicLongFieldUpdater V_TIME_UPDATER = + AtomicLongFieldUpdater.newUpdater(UniformRateLimiter.class, "virtualTime"); + private static final AtomicLongFieldUpdater START_UPDATER = + AtomicLongFieldUpdater.newUpdater(UniformRateLimiter.class, "start"); + private static final double ONE_SEC_IN_NS = TimeUnit.SECONDS.toNanos(1); + private volatile long start = Long.MIN_VALUE; + private volatile long virtualTime; + private final double opsPerSec; + private final long intervalNs; + + public UniformRateLimiter(final double opsPerSec) { + if (Double.isNaN(opsPerSec) || Double.isInfinite(opsPerSec)) { + throw new IllegalArgumentException("opsPerSec cannot be Nan or Infinite"); + } + if (opsPerSec <= 0) { + throw new IllegalArgumentException("opsPerSec must be greater then 0"); + } + this.opsPerSec = opsPerSec; + intervalNs = Math.round(ONE_SEC_IN_NS / opsPerSec); + } + + public double getOpsPerSec() { + return opsPerSec; + } + + public long getIntervalNs() { + return intervalNs; + } + + public long acquire() { + final long currOpIndex = V_TIME_UPDATER.getAndIncrement(this); + long start = this.start; + if (start == Long.MIN_VALUE) { + start = System.nanoTime(); + if (!START_UPDATER.compareAndSet(this, Long.MIN_VALUE, start)) { + start = this.start; + assert start != Long.MIN_VALUE; + } + } + return start + currOpIndex * intervalNs; + } + + public static void uninterruptibleSleepNs(final long intendedTime) { + long sleepNs; + while ((sleepNs = (intendedTime - System.nanoTime())) > 0) { + LockSupport.parkNanos(sleepNs); + } + } +} diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index c56119a1d..f74e3513a 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -200,6 +200,9 @@ public PeriodStats getPeriodStats() { stats.publishLatency.add(Histogram.decodeFromCompressedByteBuffer( ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30))); + stats.publishDelayLatency.add(Histogram.decodeFromCompressedByteBuffer( + ByteBuffer.wrap(is.publishDelayLatencyBytes), TimeUnit.SECONDS.toMicros(30))); + stats.endToEndLatency.add(Histogram.decodeFromCompressedByteBuffer( ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); } catch (ArrayIndexOutOfBoundsException | DataFormatException e) { @@ -225,6 +228,15 @@ public CumulativeLatencies getCumulativeLatencies() { throw new RuntimeException(e); } + try { + stats.publishDelayLatency.add(Histogram.decodeFromCompressedByteBuffer( + ByteBuffer.wrap(is.publishDelayLatencyBytes), TimeUnit.SECONDS.toMicros(30))); + } catch (Exception e) { + log.error("Failed to decode publish delay latency: {}", + ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.publishDelayLatencyBytes))); + throw new RuntimeException(e); + } + try { stats.endToEndLatency.add(Histogram.decodeFromCompressedByteBuffer( ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index 022f4fa2e..d3437f0cb 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -18,6 +18,7 @@ */ package io.openmessaging.benchmark.worker; +import static io.openmessaging.benchmark.utils.UniformRateLimiter.*; import static java.util.stream.Collectors.toList; import java.io.File; @@ -27,10 +28,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; -import java.util.function.Function; +import io.openmessaging.benchmark.utils.UniformRateLimiter; import org.HdrHistogram.Recorder; import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.NullStatsLogger; @@ -69,7 +71,7 @@ public class LocalWorker implements Worker, ConsumerCallback { private List producers = new ArrayList<>(); private List consumers = new ArrayList<>(); - private final RateLimiter rateLimiter = RateLimiter.create(1.0); + private volatile UniformRateLimiter rateLimiter = new UniformRateLimiter(1.0); private final ExecutorService executor = Executors.newCachedThreadPool(new DefaultThreadFactory("local-worker")); @@ -94,6 +96,10 @@ public class LocalWorker implements Worker, ConsumerCallback { private final Recorder cumulativePublishLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5); private final OpStatsLogger publishLatencyStats; + private final Recorder publishDelayLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5); + private final Recorder cumulativePublishDelayLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5); + private final OpStatsLogger publishDelayLatencyStats; + private final Recorder endToEndLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5); private final Recorder endToEndCumulativeLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5); private final OpStatsLogger endToEndLatencyStats; @@ -112,6 +118,7 @@ public LocalWorker(StatsLogger statsLogger) { StatsLogger producerStatsLogger = statsLogger.scope("producer"); this.messagesSentCounter = producerStatsLogger.getCounter("messages_sent"); this.bytesSentCounter = producerStatsLogger.getCounter("bytes_sent"); + this.publishDelayLatencyStats = producerStatsLogger.getOpStatsLogger("producer_delay_latency"); this.publishLatencyStats = producerStatsLogger.getOpStatsLogger("produce_latency"); StatsLogger consumerStatsLogger = statsLogger.scope("consumer"); @@ -184,7 +191,7 @@ public void createConsumers(ConsumerAssignment consumerAssignment) { public void startLoad(ProducerWorkAssignment producerWorkAssignment) { int processors = Runtime.getRuntime().availableProcessors(); - rateLimiter.setRate(producerWorkAssignment.publishRate); + rateLimiter = new UniformRateLimiter(producerWorkAssignment.publishRate); Map> processorAssignment = new TreeMap<>(); @@ -208,14 +215,15 @@ public void probeProducers() throws IOException { private void submitProducersToExecutor(List producers, KeyDistributor keyDistributor, List payloads) { executor.submit(() -> { int payloadCount = payloads.size(); - Random r = new Random(); + ThreadLocalRandom r = ThreadLocalRandom.current(); byte[] firstPayload = payloads.get(0); try { while (!testCompleted) { producers.forEach(producer -> { - rateLimiter.acquire(); byte[] payloadData = payloadCount == 0 ? firstPayload : payloads.get(r.nextInt(payloadCount)); + final long intendedSendTime = rateLimiter.acquire(); + uninterruptibleSleepNs(intendedSendTime); final long sendTime = System.nanoTime(); producer.sendAsync(Optional.ofNullable(keyDistributor.next()), payloadData) .thenRun(() -> { @@ -229,6 +237,11 @@ private void submitProducersToExecutor(List producers, KeyDis publishLatencyRecorder.recordValue(latencyMicros); cumulativePublishLatencyRecorder.recordValue(latencyMicros); publishLatencyStats.registerSuccessfulEvent(latencyMicros, TimeUnit.MICROSECONDS); + + final long sendDelayMicros = TimeUnit.NANOSECONDS.toMicros(sendTime - intendedSendTime); + publishDelayLatencyRecorder.recordValue(sendDelayMicros); + cumulativePublishDelayLatencyRecorder.recordValue(sendDelayMicros); + publishDelayLatencyStats.registerSuccessfulEvent(sendDelayMicros, TimeUnit.MICROSECONDS); }).exceptionally(ex -> { log.warn("Write error on message", ex); return null; @@ -244,10 +257,10 @@ private void submitProducersToExecutor(List producers, KeyDis @Override public void adjustPublishRate(double publishRate) { if(publishRate < 1.0) { - rateLimiter.setRate(1.0); + rateLimiter = new UniformRateLimiter(1.0); return; } - rateLimiter.setRate(publishRate); + rateLimiter = new UniformRateLimiter(publishRate); } @Override @@ -264,6 +277,7 @@ public PeriodStats getPeriodStats() { stats.totalMessagesReceived = totalMessagesReceived.sum(); stats.publishLatency = publishLatencyRecorder.getIntervalHistogram(); + stats.publishDelayLatency = publishDelayLatencyRecorder.getIntervalHistogram(); stats.endToEndLatency = endToEndLatencyRecorder.getIntervalHistogram(); return stats; } @@ -272,6 +286,7 @@ public PeriodStats getPeriodStats() { public CumulativeLatencies getCumulativeLatencies() { CumulativeLatencies latencies = new CumulativeLatencies(); latencies.publishLatency = cumulativePublishLatencyRecorder.getIntervalHistogram(); + latencies.publishDelayLatency = cumulativePublishDelayLatencyRecorder.getIntervalHistogram(); latencies.endToEndLatency = endToEndCumulativeLatencyRecorder.getIntervalHistogram(); return latencies; } @@ -334,6 +349,8 @@ public void resumeConsumers() throws IOException { public void resetStats() throws IOException { publishLatencyRecorder.reset(); cumulativePublishLatencyRecorder.reset(); + publishDelayLatencyRecorder.reset(); + cumulativePublishDelayLatencyRecorder.reset(); endToEndLatencyRecorder.reset(); endToEndCumulativeLatencyRecorder.reset(); } @@ -345,6 +362,8 @@ public void stopAll() throws IOException { publishLatencyRecorder.reset(); cumulativePublishLatencyRecorder.reset(); + publishDelayLatencyRecorder.reset(); + cumulativePublishDelayLatencyRecorder.reset(); endToEndLatencyRecorder.reset(); endToEndCumulativeLatencyRecorder.reset(); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java index dcf63fdda..0db00aab9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java @@ -135,6 +135,12 @@ private void handlePeriodStats(Context ctx) throws Exception { histogramSerializationBuffer.flip(); histogramSerializationBuffer.get(stats.publishLatencyBytes); + histogramSerializationBuffer.clear(); + stats.publishDelayLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); + stats.publishDelayLatencyBytes = new byte[histogramSerializationBuffer.position()]; + histogramSerializationBuffer.flip(); + histogramSerializationBuffer.get(stats.publishDelayLatencyBytes); + histogramSerializationBuffer.clear(); stats.endToEndLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); stats.endToEndLatencyBytes = new byte[histogramSerializationBuffer.position()]; @@ -156,6 +162,12 @@ private void handleCumulativeLatencies(Context ctx) throws Exception { histogramSerializationBuffer.flip(); histogramSerializationBuffer.get(stats.publishLatencyBytes); + histogramSerializationBuffer.clear(); + stats.publishDelayLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); + stats.publishDelayLatencyBytes = new byte[histogramSerializationBuffer.position()]; + histogramSerializationBuffer.flip(); + histogramSerializationBuffer.get(stats.publishDelayLatencyBytes); + histogramSerializationBuffer.clear(); stats.endToEndLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); stats.endToEndLatencyBytes = new byte[histogramSerializationBuffer.position()]; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java index 7cff435ff..cc5e5b6bd 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java @@ -30,6 +30,10 @@ public class CumulativeLatencies { public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); public byte[] publishLatencyBytes; + @JsonIgnore + public Histogram publishDelayLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); + public byte[] publishDelayLatencyBytes; + @JsonIgnore public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5); public byte[] endToEndLatencyBytes; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java index f5b5704ab..9ce4ff33f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java @@ -38,6 +38,11 @@ public class PeriodStats { public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); public byte[] publishLatencyBytes; + @JsonIgnore + public Histogram publishDelayLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); + public byte[] publishDelayLatencyBytes; + + @JsonIgnore public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5); public byte[] endToEndLatencyBytes; diff --git a/bin/create_charts.py b/bin/create_charts.py index 54391abd1..fe82fc315 100755 --- a/bin/create_charts.py +++ b/bin/create_charts.py @@ -43,6 +43,10 @@ def create_charts(test_results): y_label='Latency (ms)', time_series=[(x['driver'], x['publishLatency99pct']) for x in results]) + create_chart(workload, 'Publish Delay latency 99pct', + y_label='Latency (us)', + time_series=[(x['driver'], x['publishDelayLatency99pct']) for x in results]) + create_chart(workload, 'Publish rate', y_label='Rate (msg/s)', time_series=[(x['driver'], x['publishRate']) for x in results]) @@ -63,6 +67,10 @@ def create_charts(test_results): y_label='Latency (ms)', time_series=[(x['driver'], x['aggregatedPublishLatencyQuantiles']) for x in results]) + create_quantile_chart(workload, 'Publish Delay Latency Quantiles', + y_label='Latency (us)', + time_series=[(x['driver'], x['aggregatedPublishDelayLatencyQuantiles']) for x in results]) + create_quantile_chart(workload, 'End To End Latency Quantiles', y_label='Latency (ms)', time_series=[(x['driver'], x['aggregatedEndToEndLatencyQuantiles']) for x in results]) From 0209bd1e1d44bd3ba03d818a38fc66922418e589 Mon Sep 17 00:00:00 2001 From: cnScarb Date: Fri, 8 Apr 2022 10:26:17 +0800 Subject: [PATCH 015/103] Fix copyright (#243) (#263) --- benchmark-framework/pom.xml | 31 +++++++------------ .../io/openmessaging/benchmark/Benchmark.java | 25 ++++++--------- .../benchmark/DriverConfiguration.java | 25 ++++++--------- .../openmessaging/benchmark/ResultsToCsv.java | 25 ++++++--------- .../openmessaging/benchmark/TestResult.java | 25 ++++++--------- .../io/openmessaging/benchmark/Workers.java | 25 ++++++--------- .../io/openmessaging/benchmark/Workload.java | 25 ++++++--------- .../benchmark/WorkloadGenerator.java | 25 ++++++--------- .../benchmark/utils/ListPartition.java | 25 ++++++--------- .../benchmark/utils/PaddingDecimalFormat.java | 25 ++++++--------- .../benchmark/utils/RandomGenerator.java | 25 ++++++--------- .../openmessaging/benchmark/utils/Timer.java | 25 ++++++--------- .../benchmark/utils/UniformRateLimiter.java | 25 ++++++--------- .../utils/distributor/KeyDistributor.java | 25 ++++++--------- .../utils/distributor/KeyDistributorType.java | 25 ++++++--------- .../utils/distributor/KeyRoundRobin.java | 25 ++++++--------- .../utils/distributor/NoKeyDistributor.java | 25 ++++++--------- .../utils/distributor/RandomNano.java | 25 ++++++--------- .../utils/payload/FilePayloadReader.java | 25 ++++++--------- .../utils/payload/PayloadException.java | 25 ++++++--------- .../utils/payload/PayloadReader.java | 25 ++++++--------- .../benchmark/worker/BenchmarkWorker.java | 25 ++++++--------- .../worker/DistributedWorkersEnsemble.java | 25 ++++++--------- .../benchmark/worker/LocalWorker.java | 25 ++++++--------- .../benchmark/worker/Worker.java | 25 ++++++--------- .../benchmark/worker/WorkerHandler.java | 25 ++++++--------- .../worker/commands/ConsumerAssignment.java | 25 ++++++--------- .../worker/commands/CountersStats.java | 25 ++++++--------- .../worker/commands/CumulativeLatencies.java | 25 ++++++--------- .../worker/commands/PeriodStats.java | 25 ++++++--------- .../commands/ProducerWorkAssignment.java | 25 ++++++--------- .../worker/commands/TopicSubscription.java | 25 ++++++--------- .../benchmark/worker/commands/TopicsInfo.java | 25 ++++++--------- .../src/main/resources/log4j2.yaml | 23 ++++++-------- bin/benchmark | 23 ++++++-------- bin/benchmark-worker | 23 ++++++-------- bin/create_charts.py | 23 ++++++-------- .../kubernetes/helm/benchmark/Chart.yaml | 23 ++++++-------- .../benchmark/templates/benchmark-worker.yaml | 23 ++++++-------- .../helm/benchmark/templates/benchmark.yaml | 23 ++++++-------- .../kubernetes/helm/benchmark/values.yaml | 23 ++++++-------- dev/license-header.txt | 23 ++++++-------- docker/Dockerfile | 23 ++++++-------- docker/Dockerfile.build | 23 ++++++-------- docker/pom.xml | 27 ++++++---------- driver-api/pom.xml | 27 ++++++---------- .../benchmark/driver/BenchmarkConsumer.java | 25 ++++++--------- .../benchmark/driver/BenchmarkDriver.java | 25 ++++++--------- .../benchmark/driver/BenchmarkProducer.java | 25 ++++++--------- .../benchmark/driver/ConsumerCallback.java | 25 ++++++--------- driver-artemis/artemis.yaml | 23 ++++++-------- driver-artemis/pom.xml | 31 +++++++------------ .../artemis/ArtemisBenchmarkConsumer.java | 25 ++++++--------- .../artemis/ArtemisBenchmarkDriver.java | 25 ++++++--------- .../artemis/ArtemisBenchmarkProducer.java | 25 ++++++--------- .../driver/artemis/ArtemisConfig.java | 25 ++++++--------- driver-bookkeeper/bookkeeper.yaml | 23 ++++++-------- driver-bookkeeper/deploy/deploy.yaml | 23 ++++++-------- .../deploy/templates/bookkeeper.conf | 23 ++++++-------- .../deploy/templates/prometheus.yml | 23 ++++++-------- .../deploy/templates/pulsar_env.sh | 23 ++++++-------- .../deploy/templates/workers.yaml | 23 ++++++-------- driver-bookkeeper/deploy/templates/zoo.cfg | 23 ++++++-------- driver-bookkeeper/pom.xml | 31 +++++++------------ .../benchmark/driver/bookkeeper/Config.java | 25 ++++++--------- .../bookkeeper/DlogBenchmarkConsumer.java | 25 ++++++--------- .../bookkeeper/DlogBenchmarkDriver.java | 25 ++++++--------- .../bookkeeper/DlogBenchmarkProducer.java | 25 ++++++--------- .../bookkeeper/stats/CounterAdaptor.java | 25 ++++++--------- .../driver/bookkeeper/stats/GaugeAdaptor.java | 25 ++++++--------- .../stats/OpStatsLoggerAdaptor.java | 25 ++++++--------- .../bookkeeper/stats/StatsLoggerAdaptor.java | 25 ++++++--------- driver-jms/artemis-jms-transactions.yaml | 23 ++++++-------- driver-jms/artemis-jms.yaml | 23 ++++++-------- driver-jms/kafka-jms.yaml | 23 ++++++-------- driver-jms/package-kafka.sh | 23 ++++++-------- driver-jms/package-pulsar.sh | 23 ++++++-------- driver-jms/pom.xml | 27 ++++++---------- driver-jms/pulsar-jms-selectors.yaml | 23 ++++++-------- driver-jms/pulsar-jms-transactions.yaml | 23 ++++++-------- driver-jms/pulsar-jms.yaml | 23 ++++++-------- .../driver/jms/JMSBenchmarkConsumer.java | 25 ++++++--------- .../driver/jms/JMSBenchmarkDriver.java | 25 ++++++--------- .../driver/jms/JMSBenchmarkProducer.java | 25 ++++++--------- .../jms/JMSBenchmarkTransactionProducer.java | 25 ++++++--------- .../driver/jms/config/JMSConfig.java | 25 ++++++--------- .../hdd-deployment/alicloud/deploy.yaml | 23 ++++++-------- .../deploy/hdd-deployment/deploy.yaml | 23 ++++++-------- .../hdd-deployment/templates/chrony.conf | 23 ++++++-------- .../templates/server.properties | 23 ++++++-------- .../hdd-deployment/templates/workers.yaml | 23 ++++++-------- .../templates/zookeeper.properties | 23 ++++++-------- .../ssd-deployment/alicloud/deploy.yaml | 23 ++++++-------- .../deploy/ssd-deployment/deploy.yaml | 23 ++++++-------- .../ssd-deployment/templates/chrony.conf | 23 ++++++-------- .../templates/server.properties | 23 ++++++-------- .../ssd-deployment/templates/workers.yaml | 23 ++++++-------- .../templates/zookeeper.properties | 23 ++++++-------- driver-kafka/kafka-big-batches-gzip.yaml | 23 ++++++-------- driver-kafka/kafka-big-batches-lz4.yaml | 23 ++++++-------- driver-kafka/kafka-big-batches-snappy.yaml | 23 ++++++-------- driver-kafka/kafka-big-batches-zstd.yaml | 23 ++++++-------- driver-kafka/kafka-big-batches.yaml | 23 ++++++-------- driver-kafka/kafka-compression-gzip.yaml | 23 ++++++-------- driver-kafka/kafka-compression-lz4.yaml | 23 ++++++-------- driver-kafka/kafka-compression-snappy.yaml | 23 ++++++-------- driver-kafka/kafka-compression-zstd.yaml | 23 ++++++-------- driver-kafka/kafka-exactly-once.yaml | 23 ++++++-------- driver-kafka/kafka-latency.yaml | 23 ++++++-------- driver-kafka/kafka-no-linger.yaml | 23 ++++++-------- driver-kafka/kafka-sync.yaml | 23 ++++++-------- driver-kafka/kafka-throughput.yaml | 23 ++++++-------- driver-kafka/pom.xml | 27 ++++++---------- .../benchmark/driver/kafka/Config.java | 25 ++++++--------- .../driver/kafka/KafkaBenchmarkConsumer.java | 25 ++++++--------- .../driver/kafka/KafkaBenchmarkDriver.java | 25 ++++++--------- .../driver/kafka/KafkaBenchmarkProducer.java | 25 ++++++--------- driver-kop/kafka_to_kafka.yaml | 23 ++++++-------- driver-kop/kafka_to_pulsar.yaml | 23 ++++++-------- driver-kop/pom.xml | 31 +++++++------------ driver-kop/pulsar_to_kafka.yaml | 23 ++++++-------- .../driver/kop/KopBenchmarkDriver.java | 25 ++++++--------- .../driver/kop/config/ClientType.java | 25 ++++++--------- .../benchmark/driver/kop/config/Config.java | 25 ++++++--------- .../driver/kop/config/PulsarConfig.java | 25 ++++++--------- .../driver/kop/KopBenchmarkDriverTest.java | 25 ++++++--------- driver-kop/src/test/resources/kop_custom.yaml | 23 ++++++-------- .../src/test/resources/kop_required.yaml | 23 ++++++-------- .../resources/kop_wrong_kafka_config.yaml | 23 ++++++-------- driver-nats-streaming/deploy/deploy.yaml | 23 ++++++-------- .../deploy/templates/cluster.conf | 23 ++++++-------- .../deploy/templates/workers.yaml | 23 ++++++-------- driver-nats-streaming/nats-streaming.yaml | 23 ++++++-------- driver-nats-streaming/pom.xml | 27 ++++++---------- .../NatsStreamingBenchmarkConsumer.java | 25 ++++++--------- .../NatsStreamingBenchmarkDriver.java | 25 ++++++--------- .../NatsStreamingBenchmarkProducer.java | 25 ++++++--------- .../NatsStreamingClientConfig.java | 25 ++++++--------- driver-nats/deploy/deploy.yaml | 23 ++++++-------- driver-nats/deploy/templates/cluster.conf | 23 ++++++-------- driver-nats/deploy/templates/nats.yaml | 23 ++++++-------- driver-nats/deploy/templates/workers.yaml | 23 ++++++-------- driver-nats/pom.xml | 27 ++++++---------- .../driver/nats/NatsBenchmarkConsumer.java | 25 ++++++--------- .../driver/nats/NatsBenchmarkDriver.java | 25 ++++++--------- .../driver/nats/NatsBenchmarkProducer.java | 25 ++++++--------- .../benchmark/driver/nats/NatsConfig.java | 25 ++++++--------- driver-nsq/deploy/deploy.yaml | 23 ++++++-------- driver-nsq/deploy/templates/workers.yaml | 23 ++++++-------- driver-nsq/nsq.yaml | 23 ++++++-------- driver-nsq/pom.xml | 27 ++++++---------- .../driver/nsq/NsqBenchmarkConsumer.java | 25 ++++++--------- .../driver/nsq/NsqBenchmarkDriver.java | 25 ++++++--------- .../driver/nsq/NsqBenchmarkProducer.java | 25 ++++++--------- .../benchmark/driver/nsq/NsqConfig.java | 25 ++++++--------- driver-pravega/build-docker.sh | 23 ++++++-------- driver-pravega/deploy-k8s-components.sh | 23 ++++++-------- .../deploy/collect_logs_and_metrics.yaml | 23 ++++++-------- driver-pravega/deploy/deploy.yaml | 23 ++++++-------- .../open_saved_metrics/docker-compose.yml | 23 ++++++-------- .../open_saved_metrics/open_saved_metrics.sh | 27 +++++++--------- .../deploy/templates/bk_server.conf | 23 ++++++-------- driver-pravega/deploy/templates/bkenv.sh | 23 ++++++-------- driver-pravega/deploy/templates/chrony.conf | 23 ++++++-------- driver-pravega/deploy/templates/common.sh | 23 ++++++-------- .../deploy/templates/config.properties | 23 ++++++-------- .../templates/controller.config.properties | 23 ++++++-------- driver-pravega/deploy/templates/logback.xml | 31 +++++++------------ .../deploy/templates/prometheus.yml | 23 ++++++-------- driver-pravega/deploy/templates/workers.yaml | 23 ++++++-------- driver-pravega/deploy/templates/zoo.cfg | 23 ++++++-------- driver-pravega/deploy/vars.yaml | 23 ++++++-------- driver-pravega/driver-bash.sh | 23 ++++++-------- driver-pravega/pom.xml | 31 +++++++------------ driver-pravega/pravega-exactly-once.yaml | 23 ++++++-------- driver-pravega/pravega.yaml | 23 ++++++-------- .../pravega/PravegaBenchmarkConsumer.java | 25 ++++++--------- .../pravega/PravegaBenchmarkDriver.java | 25 ++++++--------- .../pravega/PravegaBenchmarkProducer.java | 25 ++++++--------- .../PravegaBenchmarkTransactionProducer.java | 25 ++++++--------- .../pravega/config/PravegaClientConfig.java | 25 ++++++--------- .../driver/pravega/config/PravegaConfig.java | 25 ++++++--------- .../pravega/config/PravegaWriterConfig.java | 25 ++++++--------- driver-pravega/uninstall.sh | 23 ++++++-------- driver-pulsar/deploy/ssd/alicloud/deploy.yaml | 23 ++++++-------- driver-pulsar/deploy/ssd/deploy.yaml | 23 ++++++-------- driver-pulsar/deploy/ssd/extra_vars.yaml | 23 ++++++-------- driver-pulsar/deploy/ssd/restart_broker.yaml | 23 ++++++-------- .../templates/bookkeeper-skip-journal.conf | 23 ++++++-------- .../deploy/ssd/templates/bookkeeper.conf | 23 ++++++-------- .../deploy/ssd/templates/broker.conf | 23 ++++++-------- .../deploy/ssd/templates/chrony.conf | 23 ++++++-------- .../deploy/ssd/templates/client.conf | 23 ++++++-------- driver-pulsar/deploy/ssd/templates/kop.conf | 23 ++++++-------- driver-pulsar/deploy/ssd/templates/mop.conf | 23 ++++++-------- .../deploy/ssd/templates/prometheus.yml | 23 ++++++-------- .../deploy/ssd/templates/pulsar_env.sh | 23 ++++++-------- .../deploy/ssd/templates/workers.yaml | 23 ++++++-------- driver-pulsar/deploy/ssd/templates/zoo.cfg | 23 ++++++-------- driver-pulsar/pom.xml | 27 ++++++---------- driver-pulsar/pulsar-effectively-once.yaml | 23 ++++++-------- driver-pulsar/pulsar.yaml | 23 ++++++-------- .../pulsar/PulsarBenchmarkConsumer.java | 25 ++++++--------- .../driver/pulsar/PulsarBenchmarkDriver.java | 25 ++++++--------- .../pulsar/PulsarBenchmarkProducer.java | 25 ++++++--------- .../pulsar/config/PulsarClientConfig.java | 25 ++++++--------- .../driver/pulsar/config/PulsarConfig.java | 25 ++++++--------- .../pulsar/config/PulsarConsumerConfig.java | 25 ++++++--------- .../pulsar/config/PulsarProducerConfig.java | 25 ++++++--------- driver-rabbitmq/deploy/alicloud/deploy.yaml | 23 ++++++-------- driver-rabbitmq/deploy/deploy.yaml | 23 ++++++-------- driver-rabbitmq/deploy/templates/chrony.conf | 23 ++++++-------- .../deploy/templates/rabbitmq-env.conf | 23 ++++++-------- .../deploy/templates/rabbitmq.yaml | 23 ++++++-------- driver-rabbitmq/deploy/templates/workers.yaml | 23 ++++++-------- driver-rabbitmq/pom.xml | 27 ++++++---------- driver-rabbitmq/rabbitmq.yaml | 23 ++++++-------- .../rabbitmq/RabbitMqBenchmarkConsumer.java | 25 ++++++--------- .../rabbitmq/RabbitMqBenchmarkDriver.java | 25 ++++++--------- .../rabbitmq/RabbitMqBenchmarkProducer.java | 25 ++++++--------- .../driver/rabbitmq/RabbitMqConfig.java | 25 ++++++--------- driver-redis/pom.xml | 31 +++++++------------ driver-redis/redis.yaml | 23 ++++++-------- .../driver/redis/RedisBenchmarkConsumer.java | 25 ++++++--------- .../driver/redis/RedisBenchmarkDriver.java | 25 ++++++--------- .../driver/redis/RedisBenchmarkProducer.java | 25 ++++++--------- .../redis/client/RedisClientConfig.java | 25 ++++++--------- driver-rocketmq/deploy/deploy.yaml | 23 ++++++-------- .../deploy/templates/rmq-broker.properties | 23 ++++++-------- .../deploy/templates/rmq-namesrv.properties | 23 ++++++-------- driver-rocketmq/deploy/templates/workers.yaml | 23 ++++++-------- driver-rocketmq/pom.xml | 31 +++++++------------ driver-rocketmq/rocketmq.yaml | 23 ++++++-------- .../rocketmq/RocketMQBenchmarkConsumer.java | 25 ++++++--------- .../rocketmq/RocketMQBenchmarkDriver.java | 25 ++++++--------- .../rocketmq/RocketMQBenchmarkProducer.java | 25 ++++++--------- .../rocketmq/client/RocketMQClientConfig.java | 25 ++++++--------- package/pom.xml | 27 ++++++---------- package/src/assemble/bin.xml | 27 ++++++---------- package/src/assemble/src.xml | 27 ++++++---------- pom.xml | 31 +++++++------------ style/copyright/Apache.xml | 20 ++++++++++++ workloads/1-topic-1-partition-100b.yaml | 23 ++++++-------- workloads/1-topic-1-partition-1kb.yaml | 23 ++++++-------- ...-topic-100-partitions-1kb-4p-4c-1000k.yaml | 23 ++++++-------- ...-topic-100-partitions-1kb-4p-4c-2000k.yaml | 23 ++++++-------- ...100-partitions-1kb-4p-4c-200k-backlog.yaml | 23 ++++++-------- ...1-topic-100-partitions-1kb-4p-4c-200k.yaml | 23 ++++++-------- ...1-topic-100-partitions-1kb-4p-4c-500k.yaml | 23 ++++++-------- workloads/1-topic-100-partitions-1kb.yaml | 23 ++++++-------- ...opic-10000-partitions-1kb-4p-4c-1000k.yaml | 23 ++++++-------- ...opic-10000-partitions-1kb-4p-4c-2000k.yaml | 23 ++++++-------- ...topic-10000-partitions-1kb-4p-4c-200k.yaml | 23 ++++++-------- ...topic-10000-partitions-1kb-4p-4c-500k.yaml | 23 ++++++-------- workloads/1-topic-16-partition-100b.yaml | 23 ++++++-------- workloads/1-topic-16-partitions-1kb.yaml | 23 ++++++-------- .../1-topic-3-partition-100b-3producers.yaml | 23 ++++++-------- workloads/1-topic-6-partition-100b.yaml | 23 ++++++-------- workloads/100-topics-1-partitions-1kb.yaml | 23 ++++++-------- workloads/1m-10-topics-1-partition-100b.yaml | 23 ++++++-------- .../1m-10-topics-16-partitions-100b.yaml | 23 ++++++-------- workloads/1m-10-topics-2-partitions-100b.yaml | 23 ++++++-------- workloads/1m-10-topics-3-partitions-100b.yaml | 23 ++++++-------- workloads/1m-10-topics-4-partitions-100b.yaml | 23 ++++++-------- workloads/1m-10-topics-6-partitions-100b.yaml | 23 ++++++-------- workloads/1m-10-topics-9-partitions-100b.yaml | 23 ++++++-------- .../backlog-1-topic-1-partition-1kb.yaml | 23 ++++++-------- .../backlog-1-topic-16-partitions-1kb.yaml | 23 ++++++-------- ...x-rate-1-topic-1-partition-1p-1c-100b.yaml | 23 ++++++-------- ...ax-rate-1-topic-1-partition-1p-1c-1kb.yaml | 23 ++++++-------- ...x-rate-1-topic-1-partition-1p-1c-64kb.yaml | 23 ++++++-------- ...ax-rate-1-topic-1-partition-4p-1c-1kb.yaml | 23 ++++++-------- ...te-1-topic-10-partitions-10p-10c-100b.yaml | 23 ++++++-------- ...ate-1-topic-10-partitions-10p-10c-1kb.yaml | 23 ++++++-------- ...te-1-topic-10-partitions-10p-10c-64kb.yaml | 23 ++++++-------- ...rate-1-topic-10-partitions-1p-1c-100b.yaml | 23 ++++++-------- ...-rate-1-topic-10-partitions-1p-1c-1kb.yaml | 23 ++++++-------- ...rate-1-topic-10-partitions-1p-1c-64kb.yaml | 23 ++++++-------- .../max-rate-1-topic-100-partitions-100b.yaml | 23 ++++++-------- ...1-topic-100-partitions-100p-100c-100b.yaml | 23 ++++++-------- ...-1-topic-100-partitions-100p-100c-1kb.yaml | 23 ++++++-------- ...1-topic-100-partitions-100p-100c-64kb.yaml | 23 ++++++-------- .../max-rate-1-topic-100-partitions-1kb.yaml | 23 ++++++-------- ...ate-1-topic-100-partitions-1p-1c-100b.yaml | 23 ++++++-------- ...rate-1-topic-100-partitions-1p-1c-1kb.yaml | 23 ++++++-------- .../max-rate-1-topic-16-partitions-100b.yaml | 23 ++++++-------- .../max-rate-1-topic-16-partitions-1kb.yaml | 23 ++++++-------- ...te-1-topic-20-partitions-20p-20c-100b.yaml | 23 ++++++-------- ...ate-1-topic-20-partitions-20p-20c-1kb.yaml | 23 ++++++-------- ...te-1-topic-20-partitions-20p-20c-64kb.yaml | 23 ++++++-------- ...te-1-topic-30-partitions-30p-30c-100b.yaml | 23 ++++++-------- ...ate-1-topic-30-partitions-30p-30c-1kb.yaml | 23 ++++++-------- ...te-1-topic-40-partitions-40p-40c-100b.yaml | 23 ++++++-------- ...ate-1-topic-40-partitions-40p-40c-1kb.yaml | 23 ++++++-------- ...te-1-topic-40-partitions-40p-40c-64kb.yaml | 23 ++++++-------- ...te-1-topic-50-partitions-50p-50c-100b.yaml | 23 ++++++-------- ...ate-1-topic-50-partitions-50p-50c-1kb.yaml | 23 ++++++-------- ...te-1-topic-60-partitions-60p-60c-100b.yaml | 23 ++++++-------- ...ate-1-topic-60-partitions-60p-60c-1kb.yaml | 23 ++++++-------- ...te-1-topic-60-partitions-60p-60c-64kb.yaml | 23 ++++++-------- ...te-1-topic-70-partitions-70p-70c-100b.yaml | 23 ++++++-------- ...ate-1-topic-70-partitions-70p-70c-1kb.yaml | 23 ++++++-------- ...te-1-topic-80-partitions-80p-80c-100b.yaml | 23 ++++++-------- ...ate-1-topic-80-partitions-80p-80c-1kb.yaml | 23 ++++++-------- ...te-1-topic-80-partitions-80p-80c-64kb.yaml | 23 ++++++-------- ...te-1-topic-90-partitions-90p-90c-100b.yaml | 23 ++++++-------- ...ate-1-topic-90-partitions-90p-90c-1kb.yaml | 23 ++++++-------- .../max-rate-10-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-10-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-10-topics-1-partition-64kb.yaml | 23 ++++++-------- .../max-rate-100-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-100-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-100-topics-1-partition-64kb.yaml | 23 ++++++-------- .../max-rate-20-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-20-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-20-topics-1-partition-64kb.yaml | 23 ++++++-------- .../max-rate-30-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-30-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-40-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-40-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-40-topics-1-partition-64kb.yaml | 23 ++++++-------- .../max-rate-50-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-50-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-60-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-60-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-60-topics-1-partition-64kb.yaml | 23 ++++++-------- .../max-rate-70-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-70-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-80-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-80-topics-1-partition-1kb.yaml | 23 ++++++-------- .../max-rate-80-topics-1-partition-64kb.yaml | 23 ++++++-------- .../max-rate-90-topics-1-partition-100b.yaml | 23 ++++++-------- .../max-rate-90-topics-1-partition-1kb.yaml | 23 ++++++-------- workloads/simple-workload.yaml | 23 ++++++-------- 334 files changed, 3158 insertions(+), 4845 deletions(-) create mode 100644 style/copyright/Apache.xml diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index a379ef4ee..9e4b75543 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -1,23 +1,16 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index 86ddd9e9c..62a4c8865 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java index 07852263b..e220cfa2b 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java index 5b09b9ec0..cb30a5646 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java index 57c140094..62e4ae396 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java index 56c5215de..52da2c530 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java index 844360e65..759eb09af 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 780175cac..2fc7bd1c3 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java index 37b679e37..2af6cfca9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java index 5fef088d1..31da515ce 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java @@ -1,21 +1,16 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java index 563377f19..2eba74b9e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java index 533ca92e6..d657daf13 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java index 2c31cb1c0..829e118e9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java index a56fc9621..1f26d22c1 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils.distributor; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java index db953a3c0..773e8d4f8 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils.distributor; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java index 749e3e864..0a3182221 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils.distributor; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java index d9c8d29ac..593f0e5d4 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils.distributor; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java index 5cc1c8f61..f7265c140 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils.distributor; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java index c9a02c35d..27cd9463f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils.payload; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java index 6abfca696..a3887ac31 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils.payload; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java index cf00c9d38..97d82aa80 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.utils.payload; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java index 3587ba81b..d6f9d52fa 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index f74e3513a..570c6c81e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index d3437f0cb..3598a5bbd 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java index d7bcb96d9..794e5635a 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java index 0db00aab9..951fab4e9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java index edb7961f6..3ae9c97e7 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker.commands; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java index 0d4c7e8ac..0da132c47 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker.commands; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java index cc5e5b6bd..92a12ef7b 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker.commands; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java index 9ce4ff33f..1d8c62b73 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker.commands; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java index ac8041dfb..53c7e8869 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker.commands; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java index de35a5211..f21ce6745 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker.commands; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java index 34bc968e5..b277a8f5f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.worker.commands; diff --git a/benchmark-framework/src/main/resources/log4j2.yaml b/benchmark-framework/src/main/resources/log4j2.yaml index d2d565e1e..0c654be37 100644 --- a/benchmark-framework/src/main/resources/log4j2.yaml +++ b/benchmark-framework/src/main/resources/log4j2.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Configuration: diff --git a/bin/benchmark b/bin/benchmark index c5452d973..9954f632d 100755 --- a/bin/benchmark +++ b/bin/benchmark @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # if [ -d "./lib" ]; then diff --git a/bin/benchmark-worker b/bin/benchmark-worker index 8447a4cc0..ce81e0c8a 100755 --- a/bin/benchmark-worker +++ b/bin/benchmark-worker @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # if [ -d "./lib" ]; then diff --git a/bin/create_charts.py b/bin/create_charts.py index fe82fc315..d326edd03 100755 --- a/bin/create_charts.py +++ b/bin/create_charts.py @@ -1,21 +1,16 @@ #!/usr/bin/env python # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/deployment/kubernetes/helm/benchmark/Chart.yaml b/deployment/kubernetes/helm/benchmark/Chart.yaml index a1dcacefd..85d0c9556 100644 --- a/deployment/kubernetes/helm/benchmark/Chart.yaml +++ b/deployment/kubernetes/helm/benchmark/Chart.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # apiVersion: v1 diff --git a/deployment/kubernetes/helm/benchmark/templates/benchmark-worker.yaml b/deployment/kubernetes/helm/benchmark/templates/benchmark-worker.yaml index bd77b7fba..250169401 100644 --- a/deployment/kubernetes/helm/benchmark/templates/benchmark-worker.yaml +++ b/deployment/kubernetes/helm/benchmark/templates/benchmark-worker.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # apiVersion: apps/v1 diff --git a/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml b/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml index 1ff03e12a..4310b79e8 100644 --- a/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml +++ b/deployment/kubernetes/helm/benchmark/templates/benchmark.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # apiVersion: v1 diff --git a/deployment/kubernetes/helm/benchmark/values.yaml b/deployment/kubernetes/helm/benchmark/values.yaml index aee1f260f..e4963a2bd 100644 --- a/deployment/kubernetes/helm/benchmark/values.yaml +++ b/deployment/kubernetes/helm/benchmark/values.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # numWorkers: 8 diff --git a/dev/license-header.txt b/dev/license-header.txt index 60b675e31..86a8beecb 100644 --- a/dev/license-header.txt +++ b/dev/license-header.txt @@ -1,16 +1,11 @@ -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index e1af6fa52..44f77bed4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # FROM openjdk:8-jdk diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build index 79c0520c9..35307f8c9 100644 --- a/docker/Dockerfile.build +++ b/docker/Dockerfile.build @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Build the Project diff --git a/docker/pom.xml b/docker/pom.xml index 3b72c5261..331cc2444 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> diff --git a/driver-api/pom.xml b/driver-api/pom.xml index 1f1cc7942..1ded7a90a 100644 --- a/driver-api/pom.xml +++ b/driver-api/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java index 0499767d6..1788dc6a5 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver; diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java index 45f20ff1a..0d986f223 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver; diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java index 5f5bf996f..da9bf06ac 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver; diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java index 24c55968a..88817009f 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver; diff --git a/driver-artemis/artemis.yaml b/driver-artemis/artemis.yaml index 3529ea61f..75dc0342f 100644 --- a/driver-artemis/artemis.yaml +++ b/driver-artemis/artemis.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Artemis diff --git a/driver-artemis/pom.xml b/driver-artemis/pom.xml index 20b0f50b9..7fd18106a 100644 --- a/driver-artemis/pom.xml +++ b/driver-artemis/pom.xml @@ -1,23 +1,16 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java index 6f6a71e64..c714e71fc 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.artemis; diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java index f547109bb..ae49824c1 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.artemis; diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java index 1db656d5e..baaa8cf7c 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.artemis; diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java index 6aed0d5ed..418340b66 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.artemis; diff --git a/driver-bookkeeper/bookkeeper.yaml b/driver-bookkeeper/bookkeeper.yaml index 8c049101e..3a6153b7f 100644 --- a/driver-bookkeeper/bookkeeper.yaml +++ b/driver-bookkeeper/bookkeeper.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: BookKeeper diff --git a/driver-bookkeeper/deploy/deploy.yaml b/driver-bookkeeper/deploy/deploy.yaml index be4df52a1..87c1af4da 100644 --- a/driver-bookkeeper/deploy/deploy.yaml +++ b/driver-bookkeeper/deploy/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Format and mount disks for BookKeeper hosts diff --git a/driver-bookkeeper/deploy/templates/bookkeeper.conf b/driver-bookkeeper/deploy/templates/bookkeeper.conf index ad8d9f937..e7ec1bdb6 100644 --- a/driver-bookkeeper/deploy/templates/bookkeeper.conf +++ b/driver-bookkeeper/deploy/templates/bookkeeper.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-bookkeeper/deploy/templates/prometheus.yml b/driver-bookkeeper/deploy/templates/prometheus.yml index 7286f762f..e32538ef1 100644 --- a/driver-bookkeeper/deploy/templates/prometheus.yml +++ b/driver-bookkeeper/deploy/templates/prometheus.yml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # my global config diff --git a/driver-bookkeeper/deploy/templates/pulsar_env.sh b/driver-bookkeeper/deploy/templates/pulsar_env.sh index 1a271b119..903b7df08 100644 --- a/driver-bookkeeper/deploy/templates/pulsar_env.sh +++ b/driver-bookkeeper/deploy/templates/pulsar_env.sh @@ -1,21 +1,16 @@ #!/usr/bin/env bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Set JAVA_HOME here to override the environment setting diff --git a/driver-bookkeeper/deploy/templates/workers.yaml b/driver-bookkeeper/deploy/templates/workers.yaml index c9ef4e456..1415c4807 100644 --- a/driver-bookkeeper/deploy/templates/workers.yaml +++ b/driver-bookkeeper/deploy/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-bookkeeper/deploy/templates/zoo.cfg b/driver-bookkeeper/deploy/templates/zoo.cfg index b33cad819..dbd337067 100644 --- a/driver-bookkeeper/deploy/templates/zoo.cfg +++ b/driver-bookkeeper/deploy/templates/zoo.cfg @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The number of milliseconds of each tick diff --git a/driver-bookkeeper/pom.xml b/driver-bookkeeper/pom.xml index 08e8a9397..fcd920f19 100644 --- a/driver-bookkeeper/pom.xml +++ b/driver-bookkeeper/pom.xml @@ -1,23 +1,16 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java index afaaedc25..04b660c50 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.bookkeeper; diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java index e2d9ae8dc..88657e6ee 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.bookkeeper; diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java index 2734eca62..b1ea6071c 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.bookkeeper; diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java index 1bf981043..ed2f667a3 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.bookkeeper; diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java index 3255eb5e8..61d359969 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.bookkeeper.stats; diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java index fe8e33105..3900ef854 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.bookkeeper.stats; diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java index 112ff7ca6..4f0c4fbdd 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.bookkeeper.stats; diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java index ad36231f4..37337557d 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.bookkeeper.stats; diff --git a/driver-jms/artemis-jms-transactions.yaml b/driver-jms/artemis-jms-transactions.yaml index 2035ee38b..099ff7518 100644 --- a/driver-jms/artemis-jms-transactions.yaml +++ b/driver-jms/artemis-jms-transactions.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: JMS diff --git a/driver-jms/artemis-jms.yaml b/driver-jms/artemis-jms.yaml index b229ef5b1..3e2ca8122 100644 --- a/driver-jms/artemis-jms.yaml +++ b/driver-jms/artemis-jms.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: JMS diff --git a/driver-jms/kafka-jms.yaml b/driver-jms/kafka-jms.yaml index c1ddce32a..d9bebc1a2 100644 --- a/driver-jms/kafka-jms.yaml +++ b/driver-jms/kafka-jms.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: JMS diff --git a/driver-jms/package-kafka.sh b/driver-jms/package-kafka.sh index 5b1d18d44..925733f5d 100755 --- a/driver-jms/package-kafka.sh +++ b/driver-jms/package-kafka.sh @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # the one argument is the path to the confluent jms client fat jar on your local system diff --git a/driver-jms/package-pulsar.sh b/driver-jms/package-pulsar.sh index 7c3195c01..cfd357d74 100755 --- a/driver-jms/package-pulsar.sh +++ b/driver-jms/package-pulsar.sh @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # cd package/target diff --git a/driver-jms/pom.xml b/driver-jms/pom.xml index 7789a27dc..f79a6462a 100644 --- a/driver-jms/pom.xml +++ b/driver-jms/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-jms/pulsar-jms-selectors.yaml b/driver-jms/pulsar-jms-selectors.yaml index a37059df6..5dbc1ee58 100644 --- a/driver-jms/pulsar-jms-selectors.yaml +++ b/driver-jms/pulsar-jms-selectors.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: JMS diff --git a/driver-jms/pulsar-jms-transactions.yaml b/driver-jms/pulsar-jms-transactions.yaml index aa0f808fa..bc4fcda41 100644 --- a/driver-jms/pulsar-jms-transactions.yaml +++ b/driver-jms/pulsar-jms-transactions.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: JMS diff --git a/driver-jms/pulsar-jms.yaml b/driver-jms/pulsar-jms.yaml index bf659d2da..6e42fbe55 100644 --- a/driver-jms/pulsar-jms.yaml +++ b/driver-jms/pulsar-jms.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: JMS diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java index 06bd2844a..cc4730d47 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.jms; diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java index c7cd77b03..5d8683000 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java @@ -1,21 +1,16 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.jms; diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java index fb651a621..7b5f9ac4b 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.jms; diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java index 9e24277f6..fc3ca70a6 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.jms; diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java index 324873aca..e550872d5 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.jms.config; diff --git a/driver-kafka/deploy/hdd-deployment/alicloud/deploy.yaml b/driver-kafka/deploy/hdd-deployment/alicloud/deploy.yaml index 8cd0ed1de..24575d447 100644 --- a/driver-kafka/deploy/hdd-deployment/alicloud/deploy.yaml +++ b/driver-kafka/deploy/hdd-deployment/alicloud/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Format and mount disks for Kafka hosts diff --git a/driver-kafka/deploy/hdd-deployment/deploy.yaml b/driver-kafka/deploy/hdd-deployment/deploy.yaml index d5d05a06c..a8d1de38d 100644 --- a/driver-kafka/deploy/hdd-deployment/deploy.yaml +++ b/driver-kafka/deploy/hdd-deployment/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Format and mount disks for Kafka hosts diff --git a/driver-kafka/deploy/hdd-deployment/templates/chrony.conf b/driver-kafka/deploy/hdd-deployment/templates/chrony.conf index 0e68c66a5..2ec5485aa 100644 --- a/driver-kafka/deploy/hdd-deployment/templates/chrony.conf +++ b/driver-kafka/deploy/hdd-deployment/templates/chrony.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Use public servers from the pool.ntp.org project. diff --git a/driver-kafka/deploy/hdd-deployment/templates/server.properties b/driver-kafka/deploy/hdd-deployment/templates/server.properties index 0d565707f..9ad5c2369 100644 --- a/driver-kafka/deploy/hdd-deployment/templates/server.properties +++ b/driver-kafka/deploy/hdd-deployment/templates/server.properties @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/deploy/hdd-deployment/templates/workers.yaml b/driver-kafka/deploy/hdd-deployment/templates/workers.yaml index c9ef4e456..1415c4807 100644 --- a/driver-kafka/deploy/hdd-deployment/templates/workers.yaml +++ b/driver-kafka/deploy/hdd-deployment/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/deploy/hdd-deployment/templates/zookeeper.properties b/driver-kafka/deploy/hdd-deployment/templates/zookeeper.properties index b33cad819..dbd337067 100644 --- a/driver-kafka/deploy/hdd-deployment/templates/zookeeper.properties +++ b/driver-kafka/deploy/hdd-deployment/templates/zookeeper.properties @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The number of milliseconds of each tick diff --git a/driver-kafka/deploy/ssd-deployment/alicloud/deploy.yaml b/driver-kafka/deploy/ssd-deployment/alicloud/deploy.yaml index 8cd0ed1de..24575d447 100644 --- a/driver-kafka/deploy/ssd-deployment/alicloud/deploy.yaml +++ b/driver-kafka/deploy/ssd-deployment/alicloud/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Format and mount disks for Kafka hosts diff --git a/driver-kafka/deploy/ssd-deployment/deploy.yaml b/driver-kafka/deploy/ssd-deployment/deploy.yaml index eaec0d276..edc5bf523 100644 --- a/driver-kafka/deploy/ssd-deployment/deploy.yaml +++ b/driver-kafka/deploy/ssd-deployment/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Format and mount disks for Kafka hosts diff --git a/driver-kafka/deploy/ssd-deployment/templates/chrony.conf b/driver-kafka/deploy/ssd-deployment/templates/chrony.conf index 0e68c66a5..2ec5485aa 100644 --- a/driver-kafka/deploy/ssd-deployment/templates/chrony.conf +++ b/driver-kafka/deploy/ssd-deployment/templates/chrony.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Use public servers from the pool.ntp.org project. diff --git a/driver-kafka/deploy/ssd-deployment/templates/server.properties b/driver-kafka/deploy/ssd-deployment/templates/server.properties index 0d565707f..9ad5c2369 100644 --- a/driver-kafka/deploy/ssd-deployment/templates/server.properties +++ b/driver-kafka/deploy/ssd-deployment/templates/server.properties @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/deploy/ssd-deployment/templates/workers.yaml b/driver-kafka/deploy/ssd-deployment/templates/workers.yaml index c9ef4e456..1415c4807 100644 --- a/driver-kafka/deploy/ssd-deployment/templates/workers.yaml +++ b/driver-kafka/deploy/ssd-deployment/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties index b33cad819..dbd337067 100644 --- a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties +++ b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The number of milliseconds of each tick diff --git a/driver-kafka/kafka-big-batches-gzip.yaml b/driver-kafka/kafka-big-batches-gzip.yaml index 603145388..7bb353d1d 100644 --- a/driver-kafka/kafka-big-batches-gzip.yaml +++ b/driver-kafka/kafka-big-batches-gzip.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-big-batches-lz4.yaml b/driver-kafka/kafka-big-batches-lz4.yaml index 0c3fcfdf6..19d7f2809 100644 --- a/driver-kafka/kafka-big-batches-lz4.yaml +++ b/driver-kafka/kafka-big-batches-lz4.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-big-batches-snappy.yaml b/driver-kafka/kafka-big-batches-snappy.yaml index f7a70161c..0b9d0f777 100644 --- a/driver-kafka/kafka-big-batches-snappy.yaml +++ b/driver-kafka/kafka-big-batches-snappy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-big-batches-zstd.yaml b/driver-kafka/kafka-big-batches-zstd.yaml index 162ff7225..b7e173d6a 100644 --- a/driver-kafka/kafka-big-batches-zstd.yaml +++ b/driver-kafka/kafka-big-batches-zstd.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-big-batches.yaml b/driver-kafka/kafka-big-batches.yaml index 5c3b99b0b..066c81cb9 100644 --- a/driver-kafka/kafka-big-batches.yaml +++ b/driver-kafka/kafka-big-batches.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-compression-gzip.yaml b/driver-kafka/kafka-compression-gzip.yaml index 68dc9dba7..fe6d92f0a 100644 --- a/driver-kafka/kafka-compression-gzip.yaml +++ b/driver-kafka/kafka-compression-gzip.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-compression-lz4.yaml b/driver-kafka/kafka-compression-lz4.yaml index f756ec8c6..1534bf1ef 100644 --- a/driver-kafka/kafka-compression-lz4.yaml +++ b/driver-kafka/kafka-compression-lz4.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-compression-snappy.yaml b/driver-kafka/kafka-compression-snappy.yaml index dcdfcfd1b..40367ea2a 100644 --- a/driver-kafka/kafka-compression-snappy.yaml +++ b/driver-kafka/kafka-compression-snappy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-compression-zstd.yaml b/driver-kafka/kafka-compression-zstd.yaml index 1a5a577fa..87581bda9 100644 --- a/driver-kafka/kafka-compression-zstd.yaml +++ b/driver-kafka/kafka-compression-zstd.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-exactly-once.yaml b/driver-kafka/kafka-exactly-once.yaml index e85d6ea1c..b65ffbfcc 100644 --- a/driver-kafka/kafka-exactly-once.yaml +++ b/driver-kafka/kafka-exactly-once.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-latency.yaml b/driver-kafka/kafka-latency.yaml index 6246c0f08..1da1cbf35 100644 --- a/driver-kafka/kafka-latency.yaml +++ b/driver-kafka/kafka-latency.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-no-linger.yaml b/driver-kafka/kafka-no-linger.yaml index 768bd7eed..e9868b351 100644 --- a/driver-kafka/kafka-no-linger.yaml +++ b/driver-kafka/kafka-no-linger.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/kafka-sync.yaml b/driver-kafka/kafka-sync.yaml index 2fa14cd01..0de4d795d 100644 --- a/driver-kafka/kafka-sync.yaml +++ b/driver-kafka/kafka-sync.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Kafka driver profile that is configuring to sync all diff --git a/driver-kafka/kafka-throughput.yaml b/driver-kafka/kafka-throughput.yaml index 4c00b39d4..48901e246 100644 --- a/driver-kafka/kafka-throughput.yaml +++ b/driver-kafka/kafka-throughput.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index 83744ea7c..902e68ddf 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java index b73eb0b68..710580344 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kafka; diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java index 61c2dc479..171c852e5 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kafka; diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java index f1ea60bee..1b1f207df 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kafka; diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java index 830df2652..7faba69c9 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kafka; diff --git a/driver-kop/kafka_to_kafka.yaml b/driver-kop/kafka_to_kafka.yaml index b5e6c14b3..d3a96a7cc 100644 --- a/driver-kop/kafka_to_kafka.yaml +++ b/driver-kop/kafka_to_kafka.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Kafka producer and Pulsar consumer diff --git a/driver-kop/kafka_to_pulsar.yaml b/driver-kop/kafka_to_pulsar.yaml index 22d28da0e..b1ecc256d 100644 --- a/driver-kop/kafka_to_pulsar.yaml +++ b/driver-kop/kafka_to_pulsar.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Kafka producer and Pulsar consumer diff --git a/driver-kop/pom.xml b/driver-kop/pom.xml index 91e014c0a..cf78b95a5 100644 --- a/driver-kop/pom.xml +++ b/driver-kop/pom.xml @@ -1,23 +1,16 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-kop/pulsar_to_kafka.yaml b/driver-kop/pulsar_to_kafka.yaml index bf5c80b36..d587395c3 100644 --- a/driver-kop/pulsar_to_kafka.yaml +++ b/driver-kop/pulsar_to_kafka.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Kafka producer and Pulsar consumer diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java index ec7038588..d855d7c84 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kop; diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java index ac71aaf04..1e25a856e 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kop.config; diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java index 2456413af..daff56a9c 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kop.config; diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java index 622d60455..d9c36bc09 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kop.config; diff --git a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java index 3717d3812..b1fbdffb2 100644 --- a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java +++ b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.kop; diff --git a/driver-kop/src/test/resources/kop_custom.yaml b/driver-kop/src/test/resources/kop_custom.yaml index cf9fd3a06..755b40e27 100644 --- a/driver-kop/src/test/resources/kop_custom.yaml +++ b/driver-kop/src/test/resources/kop_custom.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The KoP config file whose optional items are customized with different values from default. diff --git a/driver-kop/src/test/resources/kop_required.yaml b/driver-kop/src/test/resources/kop_required.yaml index 2547ee587..95548a775 100644 --- a/driver-kop/src/test/resources/kop_required.yaml +++ b/driver-kop/src/test/resources/kop_required.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The KoP config file with required config items. diff --git a/driver-kop/src/test/resources/kop_wrong_kafka_config.yaml b/driver-kop/src/test/resources/kop_wrong_kafka_config.yaml index 8baaa47a7..0fa6b4fa4 100644 --- a/driver-kop/src/test/resources/kop_wrong_kafka_config.yaml +++ b/driver-kop/src/test/resources/kop_wrong_kafka_config.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The KoP config file whose kafka config is wrong (without required configs) diff --git a/driver-nats-streaming/deploy/deploy.yaml b/driver-nats-streaming/deploy/deploy.yaml index ab0f2ab13..519e6fb57 100644 --- a/driver-nats-streaming/deploy/deploy.yaml +++ b/driver-nats-streaming/deploy/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-nats-streaming/deploy/templates/cluster.conf b/driver-nats-streaming/deploy/templates/cluster.conf index 55fdc5ef9..e794be098 100644 --- a/driver-nats-streaming/deploy/templates/cluster.conf +++ b/driver-nats-streaming/deploy/templates/cluster.conf @@ -1,21 +1,16 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Cluster Seed Node diff --git a/driver-nats-streaming/deploy/templates/workers.yaml b/driver-nats-streaming/deploy/templates/workers.yaml index c9ef4e456..1415c4807 100644 --- a/driver-nats-streaming/deploy/templates/workers.yaml +++ b/driver-nats-streaming/deploy/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-nats-streaming/nats-streaming.yaml b/driver-nats-streaming/nats-streaming.yaml index d7ab09fa6..94d7f250e 100644 --- a/driver-nats-streaming/nats-streaming.yaml +++ b/driver-nats-streaming/nats-streaming.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-nats-streaming/pom.xml b/driver-nats-streaming/pom.xml index 01720dd1c..28f1bb281 100644 --- a/driver-nats-streaming/pom.xml +++ b/driver-nats-streaming/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java index 9112efebe..8c5fd250b 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.nats; diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index 8e18c187e..dadc4d278 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.nats; diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java index 530b17880..bfcba157d 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.nats; diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java index 39cfd3318..6fb51d4a1 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.nats; diff --git a/driver-nsq/deploy/deploy.yaml b/driver-nsq/deploy/deploy.yaml index 55cbd8ea2..b7243e215 100644 --- a/driver-nsq/deploy/deploy.yaml +++ b/driver-nsq/deploy/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-nsq/deploy/templates/workers.yaml b/driver-nsq/deploy/templates/workers.yaml index c9ef4e456..1415c4807 100644 --- a/driver-nsq/deploy/templates/workers.yaml +++ b/driver-nsq/deploy/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-nsq/nsq.yaml b/driver-nsq/nsq.yaml index 86547b50f..3581b027a 100644 --- a/driver-nsq/nsq.yaml +++ b/driver-nsq/nsq.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-nsq/pom.xml b/driver-nsq/pom.xml index 2e5c505c9..5949884a5 100644 --- a/driver-nsq/pom.xml +++ b/driver-nsq/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java index 89a5849dd..a301da2ca 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.nsq; diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java index 79d417cb3..4c1581c89 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.nsq; diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java index 6db3fee5c..1ae87645e 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.nsq; diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java index 9ea5590a9..5658105e2 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.nsq; diff --git a/driver-pravega/build-docker.sh b/driver-pravega/build-docker.sh index 00613b926..b9f2fb968 100755 --- a/driver-pravega/build-docker.sh +++ b/driver-pravega/build-docker.sh @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # set -ex diff --git a/driver-pravega/deploy-k8s-components.sh b/driver-pravega/deploy-k8s-components.sh index a9c1a9329..cef4487de 100755 --- a/driver-pravega/deploy-k8s-components.sh +++ b/driver-pravega/deploy-k8s-components.sh @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # set -ex diff --git a/driver-pravega/deploy/collect_logs_and_metrics.yaml b/driver-pravega/deploy/collect_logs_and_metrics.yaml index e012cb432..70e64ffa0 100644 --- a/driver-pravega/deploy/collect_logs_and_metrics.yaml +++ b/driver-pravega/deploy/collect_logs_and_metrics.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Get current time diff --git a/driver-pravega/deploy/deploy.yaml b/driver-pravega/deploy/deploy.yaml index 46de4928f..3846fa997 100644 --- a/driver-pravega/deploy/deploy.yaml +++ b/driver-pravega/deploy/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Get common facts diff --git a/driver-pravega/deploy/open_saved_metrics/docker-compose.yml b/driver-pravega/deploy/open_saved_metrics/docker-compose.yml index 01f0774e5..f57b90c09 100644 --- a/driver-pravega/deploy/open_saved_metrics/docker-compose.yml +++ b/driver-pravega/deploy/open_saved_metrics/docker-compose.yml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # version: '2' diff --git a/driver-pravega/deploy/open_saved_metrics/open_saved_metrics.sh b/driver-pravega/deploy/open_saved_metrics/open_saved_metrics.sh index d9cfacb23..75643fd77 100755 --- a/driver-pravega/deploy/open_saved_metrics/open_saved_metrics.sh +++ b/driver-pravega/deploy/open_saved_metrics/open_saved_metrics.sh @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # set -e diff --git a/driver-pravega/deploy/templates/bk_server.conf b/driver-pravega/deploy/templates/bk_server.conf index 91bf16448..4324303a9 100755 --- a/driver-pravega/deploy/templates/bk_server.conf +++ b/driver-pravega/deploy/templates/bk_server.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The bookie server configuration are organized in sections diff --git a/driver-pravega/deploy/templates/bkenv.sh b/driver-pravega/deploy/templates/bkenv.sh index 7c3c4096d..e7250703a 100755 --- a/driver-pravega/deploy/templates/bkenv.sh +++ b/driver-pravega/deploy/templates/bkenv.sh @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Set JAVA_HOME here to override the environment setting diff --git a/driver-pravega/deploy/templates/chrony.conf b/driver-pravega/deploy/templates/chrony.conf index 0e68c66a5..2ec5485aa 100644 --- a/driver-pravega/deploy/templates/chrony.conf +++ b/driver-pravega/deploy/templates/chrony.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Use public servers from the pool.ntp.org project. diff --git a/driver-pravega/deploy/templates/common.sh b/driver-pravega/deploy/templates/common.sh index e5dd6b59b..549eded18 100644 --- a/driver-pravega/deploy/templates/common.sh +++ b/driver-pravega/deploy/templates/common.sh @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Check net.ipv6.bindv6only diff --git a/driver-pravega/deploy/templates/config.properties b/driver-pravega/deploy/templates/config.properties index 96efa778b..5628f864d 100644 --- a/driver-pravega/deploy/templates/config.properties +++ b/driver-pravega/deploy/templates/config.properties @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # ## Instructions for using this file: diff --git a/driver-pravega/deploy/templates/controller.config.properties b/driver-pravega/deploy/templates/controller.config.properties index edc65d178..c387acdd8 100644 --- a/driver-pravega/deploy/templates/controller.config.properties +++ b/driver-pravega/deploy/templates/controller.config.properties @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # ## Instructions for using this file: diff --git a/driver-pravega/deploy/templates/logback.xml b/driver-pravega/deploy/templates/logback.xml index 73cd0904f..33bec7685 100644 --- a/driver-pravega/deploy/templates/logback.xml +++ b/driver-pravega/deploy/templates/logback.xml @@ -1,24 +1,17 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> diff --git a/driver-pravega/deploy/templates/prometheus.yml b/driver-pravega/deploy/templates/prometheus.yml index 8cc93a4ac..b3717a78d 100644 --- a/driver-pravega/deploy/templates/prometheus.yml +++ b/driver-pravega/deploy/templates/prometheus.yml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # my global config diff --git a/driver-pravega/deploy/templates/workers.yaml b/driver-pravega/deploy/templates/workers.yaml index c9ef4e456..1415c4807 100644 --- a/driver-pravega/deploy/templates/workers.yaml +++ b/driver-pravega/deploy/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-pravega/deploy/templates/zoo.cfg b/driver-pravega/deploy/templates/zoo.cfg index b33cad819..dbd337067 100644 --- a/driver-pravega/deploy/templates/zoo.cfg +++ b/driver-pravega/deploy/templates/zoo.cfg @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The number of milliseconds of each tick diff --git a/driver-pravega/deploy/vars.yaml b/driver-pravega/deploy/vars.yaml index 0a37e8d9a..730eb107d 100644 --- a/driver-pravega/deploy/vars.yaml +++ b/driver-pravega/deploy/vars.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # --- diff --git a/driver-pravega/driver-bash.sh b/driver-pravega/driver-bash.sh index 8a8549821..40d28bcd2 100755 --- a/driver-pravega/driver-bash.sh +++ b/driver-pravega/driver-bash.sh @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # set -ex diff --git a/driver-pravega/pom.xml b/driver-pravega/pom.xml index eea42d368..62fac1092 100644 --- a/driver-pravega/pom.xml +++ b/driver-pravega/pom.xml @@ -1,23 +1,16 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-pravega/pravega-exactly-once.yaml b/driver-pravega/pravega-exactly-once.yaml index 58ea0f709..eee7f8a3d 100644 --- a/driver-pravega/pravega-exactly-once.yaml +++ b/driver-pravega/pravega-exactly-once.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Pravega diff --git a/driver-pravega/pravega.yaml b/driver-pravega/pravega.yaml index 6b05d795d..f7c8546f0 100644 --- a/driver-pravega/pravega.yaml +++ b/driver-pravega/pravega.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Pravega diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java index 767bdd3c9..d2b12b252 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pravega; diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java index 67b245c54..2f8eefa53 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pravega; diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java index db7615968..fd9546fd3 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pravega; diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java index d65bf2fea..8071caec3 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pravega; diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java index 2d98780b2..f1dd86b60 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pravega.config; diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java index 058872c17..0853bb4b3 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pravega.config; diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java index 344bb5cb4..d470e8d31 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pravega.config; diff --git a/driver-pravega/uninstall.sh b/driver-pravega/uninstall.sh index 32351eeb8..1cac531a4 100755 --- a/driver-pravega/uninstall.sh +++ b/driver-pravega/uninstall.sh @@ -1,21 +1,16 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # set -x diff --git a/driver-pulsar/deploy/ssd/alicloud/deploy.yaml b/driver-pulsar/deploy/ssd/alicloud/deploy.yaml index 3d9702706..f4ccd18e5 100644 --- a/driver-pulsar/deploy/ssd/alicloud/deploy.yaml +++ b/driver-pulsar/deploy/ssd/alicloud/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Prepare variable diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index 408dddd41..db28af30e 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Initialize some default values diff --git a/driver-pulsar/deploy/ssd/extra_vars.yaml b/driver-pulsar/deploy/ssd/extra_vars.yaml index 684b170c0..ee09257ad 100644 --- a/driver-pulsar/deploy/ssd/extra_vars.yaml +++ b/driver-pulsar/deploy/ssd/extra_vars.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # pulsar_version: 2.9.1 diff --git a/driver-pulsar/deploy/ssd/restart_broker.yaml b/driver-pulsar/deploy/ssd/restart_broker.yaml index ff43cc233..0c3bf98c1 100644 --- a/driver-pulsar/deploy/ssd/restart_broker.yaml +++ b/driver-pulsar/deploy/ssd/restart_broker.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Initialize variables diff --git a/driver-pulsar/deploy/ssd/templates/bookkeeper-skip-journal.conf b/driver-pulsar/deploy/ssd/templates/bookkeeper-skip-journal.conf index c86749f89..f09876a65 100644 --- a/driver-pulsar/deploy/ssd/templates/bookkeeper-skip-journal.conf +++ b/driver-pulsar/deploy/ssd/templates/bookkeeper-skip-journal.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-pulsar/deploy/ssd/templates/bookkeeper.conf b/driver-pulsar/deploy/ssd/templates/bookkeeper.conf index 1d204a6eb..dbd9631e3 100644 --- a/driver-pulsar/deploy/ssd/templates/bookkeeper.conf +++ b/driver-pulsar/deploy/ssd/templates/bookkeeper.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-pulsar/deploy/ssd/templates/broker.conf b/driver-pulsar/deploy/ssd/templates/broker.conf index be6ca2c9f..6bb8685c4 100644 --- a/driver-pulsar/deploy/ssd/templates/broker.conf +++ b/driver-pulsar/deploy/ssd/templates/broker.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-pulsar/deploy/ssd/templates/chrony.conf b/driver-pulsar/deploy/ssd/templates/chrony.conf index 0e68c66a5..2ec5485aa 100644 --- a/driver-pulsar/deploy/ssd/templates/chrony.conf +++ b/driver-pulsar/deploy/ssd/templates/chrony.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Use public servers from the pool.ntp.org project. diff --git a/driver-pulsar/deploy/ssd/templates/client.conf b/driver-pulsar/deploy/ssd/templates/client.conf index 81c65772a..a1da5d8a4 100644 --- a/driver-pulsar/deploy/ssd/templates/client.conf +++ b/driver-pulsar/deploy/ssd/templates/client.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-pulsar/deploy/ssd/templates/kop.conf b/driver-pulsar/deploy/ssd/templates/kop.conf index 6bed09cec..710593b63 100644 --- a/driver-pulsar/deploy/ssd/templates/kop.conf +++ b/driver-pulsar/deploy/ssd/templates/kop.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # kafkaListeners=PLAINTEXT://{{ hostvars[inventory_hostname].private_ip }}:9092 diff --git a/driver-pulsar/deploy/ssd/templates/mop.conf b/driver-pulsar/deploy/ssd/templates/mop.conf index 89002b63b..8b1470110 100644 --- a/driver-pulsar/deploy/ssd/templates/mop.conf +++ b/driver-pulsar/deploy/ssd/templates/mop.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # mqttListeners=mqtt://{{ hostvars[inventory_hostname].private_ip }}:1883 diff --git a/driver-pulsar/deploy/ssd/templates/prometheus.yml b/driver-pulsar/deploy/ssd/templates/prometheus.yml index 131c23790..a539eba74 100644 --- a/driver-pulsar/deploy/ssd/templates/prometheus.yml +++ b/driver-pulsar/deploy/ssd/templates/prometheus.yml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # --- diff --git a/driver-pulsar/deploy/ssd/templates/pulsar_env.sh b/driver-pulsar/deploy/ssd/templates/pulsar_env.sh index 18eb12cf6..deb69f719 100644 --- a/driver-pulsar/deploy/ssd/templates/pulsar_env.sh +++ b/driver-pulsar/deploy/ssd/templates/pulsar_env.sh @@ -1,21 +1,16 @@ #!/usr/bin/env bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Set JAVA_HOME here to override the environment setting diff --git a/driver-pulsar/deploy/ssd/templates/workers.yaml b/driver-pulsar/deploy/ssd/templates/workers.yaml index c9ef4e456..1415c4807 100644 --- a/driver-pulsar/deploy/ssd/templates/workers.yaml +++ b/driver-pulsar/deploy/ssd/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-pulsar/deploy/ssd/templates/zoo.cfg b/driver-pulsar/deploy/ssd/templates/zoo.cfg index b33cad819..dbd337067 100644 --- a/driver-pulsar/deploy/ssd/templates/zoo.cfg +++ b/driver-pulsar/deploy/ssd/templates/zoo.cfg @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # The number of milliseconds of each tick diff --git a/driver-pulsar/pom.xml b/driver-pulsar/pom.xml index 768c6c57a..2b7f10c0c 100644 --- a/driver-pulsar/pom.xml +++ b/driver-pulsar/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-pulsar/pulsar-effectively-once.yaml b/driver-pulsar/pulsar-effectively-once.yaml index c89a93138..ec3e8e07e 100644 --- a/driver-pulsar/pulsar-effectively-once.yaml +++ b/driver-pulsar/pulsar-effectively-once.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Pulsar-effectively-once diff --git a/driver-pulsar/pulsar.yaml b/driver-pulsar/pulsar.yaml index 25846a091..6a10e8055 100644 --- a/driver-pulsar/pulsar.yaml +++ b/driver-pulsar/pulsar.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Pulsar diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java index 19421393b..f64db5f62 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pulsar; diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java index 82d966c1e..0d085dba0 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java @@ -1,21 +1,16 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pulsar; diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java index 475511c26..cad7a4a12 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pulsar; diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java index 04c828fd5..a143e75a0 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pulsar.config; diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java index e9ffb50d3..8bcb24622 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pulsar.config; diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java index 0cac20559..981271eec 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pulsar.config; diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java index 73b015d0d..0cfa555f4 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.pulsar.config; diff --git a/driver-rabbitmq/deploy/alicloud/deploy.yaml b/driver-rabbitmq/deploy/alicloud/deploy.yaml index a4b13405a..145df6b92 100644 --- a/driver-rabbitmq/deploy/alicloud/deploy.yaml +++ b/driver-rabbitmq/deploy/alicloud/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Prepare variable diff --git a/driver-rabbitmq/deploy/deploy.yaml b/driver-rabbitmq/deploy/deploy.yaml index 2d37bca0d..e9ee13b70 100644 --- a/driver-rabbitmq/deploy/deploy.yaml +++ b/driver-rabbitmq/deploy/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Format and mount disks for RabbitMQ hosts diff --git a/driver-rabbitmq/deploy/templates/chrony.conf b/driver-rabbitmq/deploy/templates/chrony.conf index 0e68c66a5..2ec5485aa 100644 --- a/driver-rabbitmq/deploy/templates/chrony.conf +++ b/driver-rabbitmq/deploy/templates/chrony.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # Use public servers from the pool.ntp.org project. diff --git a/driver-rabbitmq/deploy/templates/rabbitmq-env.conf b/driver-rabbitmq/deploy/templates/rabbitmq-env.conf index e34b66033..443cf7f97 100644 --- a/driver-rabbitmq/deploy/templates/rabbitmq-env.conf +++ b/driver-rabbitmq/deploy/templates/rabbitmq-env.conf @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # MNESIA_BASE=/mnt/data diff --git a/driver-rabbitmq/deploy/templates/rabbitmq.yaml b/driver-rabbitmq/deploy/templates/rabbitmq.yaml index e181768c8..bf8f8725e 100644 --- a/driver-rabbitmq/deploy/templates/rabbitmq.yaml +++ b/driver-rabbitmq/deploy/templates/rabbitmq.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-rabbitmq/deploy/templates/workers.yaml b/driver-rabbitmq/deploy/templates/workers.yaml index c9ef4e456..1415c4807 100644 --- a/driver-rabbitmq/deploy/templates/workers.yaml +++ b/driver-rabbitmq/deploy/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-rabbitmq/pom.xml b/driver-rabbitmq/pom.xml index 5685e0d4a..e30c157d5 100644 --- a/driver-rabbitmq/pom.xml +++ b/driver-rabbitmq/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/driver-rabbitmq/rabbitmq.yaml b/driver-rabbitmq/rabbitmq.yaml index 7651b6425..b41078eca 100644 --- a/driver-rabbitmq/rabbitmq.yaml +++ b/driver-rabbitmq/rabbitmq.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java index 4120f5cf9..975121e18 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.rabbitmq; diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index 6b411fc5e..328bd1d0f 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.rabbitmq; diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java index bbd117e02..f795158ee 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.rabbitmq; diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java index 14b3d7155..6518e44fe 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.rabbitmq; diff --git a/driver-redis/pom.xml b/driver-redis/pom.xml index 2942da669..105a07c44 100644 --- a/driver-redis/pom.xml +++ b/driver-redis/pom.xml @@ -1,23 +1,16 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> diff --git a/driver-redis/redis.yaml b/driver-redis/redis.yaml index d63c37960..cca084a40 100644 --- a/driver-redis/redis.yaml +++ b/driver-redis/redis.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Redis diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java index b3da91975..c9d8b60a2 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.redis; diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java index ed1a28552..9e5b28c87 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.redis; diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java index d98d664f9..d62c4c246 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.redis; diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java index e5e6905c8..899f752ef 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.redis.client; diff --git a/driver-rocketmq/deploy/deploy.yaml b/driver-rocketmq/deploy/deploy.yaml index e53594a2b..5278cdca6 100644 --- a/driver-rocketmq/deploy/deploy.yaml +++ b/driver-rocketmq/deploy/deploy.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # - name: Hosts addresses diff --git a/driver-rocketmq/deploy/templates/rmq-broker.properties b/driver-rocketmq/deploy/templates/rmq-broker.properties index 96574aabf..e79bec68b 100644 --- a/driver-rocketmq/deploy/templates/rmq-broker.properties +++ b/driver-rocketmq/deploy/templates/rmq-broker.properties @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # brokerClusterName=DefaultCluster diff --git a/driver-rocketmq/deploy/templates/rmq-namesrv.properties b/driver-rocketmq/deploy/templates/rmq-namesrv.properties index d8a500d9d..3663c15d8 100644 --- a/driver-rocketmq/deploy/templates/rmq-namesrv.properties +++ b/driver-rocketmq/deploy/templates/rmq-namesrv.properties @@ -1,19 +1,14 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-rocketmq/deploy/templates/workers.yaml b/driver-rocketmq/deploy/templates/workers.yaml index 7d3bbe9e0..c0dd19380 100644 --- a/driver-rocketmq/deploy/templates/workers.yaml +++ b/driver-rocketmq/deploy/templates/workers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # diff --git a/driver-rocketmq/pom.xml b/driver-rocketmq/pom.xml index 1e789fdac..cae20cadc 100644 --- a/driver-rocketmq/pom.xml +++ b/driver-rocketmq/pom.xml @@ -1,23 +1,16 @@ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> diff --git a/driver-rocketmq/rocketmq.yaml b/driver-rocketmq/rocketmq.yaml index b30dc4dc0..e075120ee 100644 --- a/driver-rocketmq/rocketmq.yaml +++ b/driver-rocketmq/rocketmq.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: RocketMQ diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java index 5935bef2b..a79b9ede1 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.rocketmq; diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java index ccb1202e8..10fd68d06 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.rocketmq; diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java index d1008c30f..838808729 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.rocketmq; diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java index fbfaa32d2..f9de02d7e 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java @@ -1,20 +1,15 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.openmessaging.benchmark.driver.rocketmq.client; diff --git a/package/pom.xml b/package/pom.xml index dc6ab7fc9..aed395e9f 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/package/src/assemble/bin.xml b/package/src/assemble/bin.xml index 1f9a8f894..19cece2ad 100644 --- a/package/src/assemble/bin.xml +++ b/package/src/assemble/bin.xml @@ -1,23 +1,16 @@ + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> 4.0.0 diff --git a/style/copyright/Apache.xml b/style/copyright/Apache.xml new file mode 100644 index 000000000..8301fbd80 --- /dev/null +++ b/style/copyright/Apache.xml @@ -0,0 +1,20 @@ + + + + + + \ No newline at end of file diff --git a/workloads/1-topic-1-partition-100b.yaml b/workloads/1-topic-1-partition-100b.yaml index 4f4f55728..1a2b4b2f7 100644 --- a/workloads/1-topic-1-partition-100b.yaml +++ b/workloads/1-topic-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 topic / 1 partition / 100b diff --git a/workloads/1-topic-1-partition-1kb.yaml b/workloads/1-topic-1-partition-1kb.yaml index caa1f204a..7c074d9a0 100644 --- a/workloads/1-topic-1-partition-1kb.yaml +++ b/workloads/1-topic-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 topic / 1 partition / 1Kb diff --git a/workloads/1-topic-100-partitions-1kb-4p-4c-1000k.yaml b/workloads/1-topic-100-partitions-1kb-4p-4c-1000k.yaml index 0eafe5fd5..95201382d 100644 --- a/workloads/1-topic-100-partitions-1kb-4p-4c-1000k.yaml +++ b/workloads/1-topic-100-partitions-1kb-4p-4c-1000k.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1000k rate 4 producers and 4 consumers on 1 topic / 100 partition diff --git a/workloads/1-topic-100-partitions-1kb-4p-4c-2000k.yaml b/workloads/1-topic-100-partitions-1kb-4p-4c-2000k.yaml index 761d172ca..2b44b5341 100644 --- a/workloads/1-topic-100-partitions-1kb-4p-4c-2000k.yaml +++ b/workloads/1-topic-100-partitions-1kb-4p-4c-2000k.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 2000k rate 4 producers and 4 consumers on 1 topic / 100 partition diff --git a/workloads/1-topic-100-partitions-1kb-4p-4c-200k-backlog.yaml b/workloads/1-topic-100-partitions-1kb-4p-4c-200k-backlog.yaml index 6918f74b9..e61149a8f 100644 --- a/workloads/1-topic-100-partitions-1kb-4p-4c-200k-backlog.yaml +++ b/workloads/1-topic-100-partitions-1kb-4p-4c-200k-backlog.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Backlog for 200k rate 4 producers and 4 consumers on 1 topic / 100 partition diff --git a/workloads/1-topic-100-partitions-1kb-4p-4c-200k.yaml b/workloads/1-topic-100-partitions-1kb-4p-4c-200k.yaml index 6993ba4c7..cfb3b4c4f 100644 --- a/workloads/1-topic-100-partitions-1kb-4p-4c-200k.yaml +++ b/workloads/1-topic-100-partitions-1kb-4p-4c-200k.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 200k rate 4 producers and 4 consumers on 1 topic / 100 partition diff --git a/workloads/1-topic-100-partitions-1kb-4p-4c-500k.yaml b/workloads/1-topic-100-partitions-1kb-4p-4c-500k.yaml index a224bed2b..a938e3652 100644 --- a/workloads/1-topic-100-partitions-1kb-4p-4c-500k.yaml +++ b/workloads/1-topic-100-partitions-1kb-4p-4c-500k.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 500k rate 4 producers and 4 consumers on 1 topic / 100 partition diff --git a/workloads/1-topic-100-partitions-1kb.yaml b/workloads/1-topic-100-partitions-1kb.yaml index 51778f8fe..6c8cff632 100644 --- a/workloads/1-topic-100-partitions-1kb.yaml +++ b/workloads/1-topic-100-partitions-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 producer / 1 consumers on 1 topic diff --git a/workloads/1-topic-10000-partitions-1kb-4p-4c-1000k.yaml b/workloads/1-topic-10000-partitions-1kb-4p-4c-1000k.yaml index 9060381f1..be14a5f75 100644 --- a/workloads/1-topic-10000-partitions-1kb-4p-4c-1000k.yaml +++ b/workloads/1-topic-10000-partitions-1kb-4p-4c-1000k.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1000k rate 4 producers and 4 consumers on 1 topic / 10000 partition diff --git a/workloads/1-topic-10000-partitions-1kb-4p-4c-2000k.yaml b/workloads/1-topic-10000-partitions-1kb-4p-4c-2000k.yaml index 93980b5ef..b5296245b 100644 --- a/workloads/1-topic-10000-partitions-1kb-4p-4c-2000k.yaml +++ b/workloads/1-topic-10000-partitions-1kb-4p-4c-2000k.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 2000k rate 4 producers and 4 consumers on 1 topic / 10000 partition diff --git a/workloads/1-topic-10000-partitions-1kb-4p-4c-200k.yaml b/workloads/1-topic-10000-partitions-1kb-4p-4c-200k.yaml index 8a851fe6b..b0b709e3c 100644 --- a/workloads/1-topic-10000-partitions-1kb-4p-4c-200k.yaml +++ b/workloads/1-topic-10000-partitions-1kb-4p-4c-200k.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 200k rate 4 producers and 4 consumers on 1 topic / 10000 partition diff --git a/workloads/1-topic-10000-partitions-1kb-4p-4c-500k.yaml b/workloads/1-topic-10000-partitions-1kb-4p-4c-500k.yaml index 0ed1cc493..cb5550f56 100644 --- a/workloads/1-topic-10000-partitions-1kb-4p-4c-500k.yaml +++ b/workloads/1-topic-10000-partitions-1kb-4p-4c-500k.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 500k rate 4 producers and 4 consumers on 1 topic / 10000 partition diff --git a/workloads/1-topic-16-partition-100b.yaml b/workloads/1-topic-16-partition-100b.yaml index 383a8a4ca..3f94b60e0 100644 --- a/workloads/1-topic-16-partition-100b.yaml +++ b/workloads/1-topic-16-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 topic / 16 partition / 100b diff --git a/workloads/1-topic-16-partitions-1kb.yaml b/workloads/1-topic-16-partitions-1kb.yaml index 56c9a3654..ed1856a79 100644 --- a/workloads/1-topic-16-partitions-1kb.yaml +++ b/workloads/1-topic-16-partitions-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 producer / 1 consumers on 1 topic diff --git a/workloads/1-topic-3-partition-100b-3producers.yaml b/workloads/1-topic-3-partition-100b-3producers.yaml index 96b160ed3..4bd561425 100644 --- a/workloads/1-topic-3-partition-100b-3producers.yaml +++ b/workloads/1-topic-3-partition-100b-3producers.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 topic / 3 partition / 100b / 3 producers diff --git a/workloads/1-topic-6-partition-100b.yaml b/workloads/1-topic-6-partition-100b.yaml index 8cc7c9b99..510d2af02 100644 --- a/workloads/1-topic-6-partition-100b.yaml +++ b/workloads/1-topic-6-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 topic / 6 partition / 100b diff --git a/workloads/100-topics-1-partitions-1kb.yaml b/workloads/100-topics-1-partitions-1kb.yaml index 379bf553e..1fa0aecc8 100644 --- a/workloads/100-topics-1-partitions-1kb.yaml +++ b/workloads/100-topics-1-partitions-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 producer / 3 consumers on 100 topics diff --git a/workloads/1m-10-topics-1-partition-100b.yaml b/workloads/1m-10-topics-1-partition-100b.yaml index cca18bdba..0168734af 100644 --- a/workloads/1m-10-topics-1-partition-100b.yaml +++ b/workloads/1m-10-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1m-10-topics-1-partition-100b diff --git a/workloads/1m-10-topics-16-partitions-100b.yaml b/workloads/1m-10-topics-16-partitions-100b.yaml index 2957f65d7..be3d6584d 100644 --- a/workloads/1m-10-topics-16-partitions-100b.yaml +++ b/workloads/1m-10-topics-16-partitions-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1m-10-topics-16-partitions-100b diff --git a/workloads/1m-10-topics-2-partitions-100b.yaml b/workloads/1m-10-topics-2-partitions-100b.yaml index 01afc5c84..e6ef2eb81 100644 --- a/workloads/1m-10-topics-2-partitions-100b.yaml +++ b/workloads/1m-10-topics-2-partitions-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1m-10-topics-2-partitions-100b diff --git a/workloads/1m-10-topics-3-partitions-100b.yaml b/workloads/1m-10-topics-3-partitions-100b.yaml index a3c2390a2..73dbb9207 100644 --- a/workloads/1m-10-topics-3-partitions-100b.yaml +++ b/workloads/1m-10-topics-3-partitions-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1m-10-topics-3-partitions-100b diff --git a/workloads/1m-10-topics-4-partitions-100b.yaml b/workloads/1m-10-topics-4-partitions-100b.yaml index a0d300260..2382bf6eb 100644 --- a/workloads/1m-10-topics-4-partitions-100b.yaml +++ b/workloads/1m-10-topics-4-partitions-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1m-10-topics-4-partitions-100b diff --git a/workloads/1m-10-topics-6-partitions-100b.yaml b/workloads/1m-10-topics-6-partitions-100b.yaml index 6ec47c434..b3dc0c3cb 100644 --- a/workloads/1m-10-topics-6-partitions-100b.yaml +++ b/workloads/1m-10-topics-6-partitions-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1m-10-topics-6-partitions-100b diff --git a/workloads/1m-10-topics-9-partitions-100b.yaml b/workloads/1m-10-topics-9-partitions-100b.yaml index 1d4d5da28..645224efc 100644 --- a/workloads/1m-10-topics-9-partitions-100b.yaml +++ b/workloads/1m-10-topics-9-partitions-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1m-10-topics-9-partitions-100b diff --git a/workloads/backlog-1-topic-1-partition-1kb.yaml b/workloads/backlog-1-topic-1-partition-1kb.yaml index 925ccaf8f..be1638ed1 100644 --- a/workloads/backlog-1-topic-1-partition-1kb.yaml +++ b/workloads/backlog-1-topic-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 producer / 1 consumers on 1 topic diff --git a/workloads/backlog-1-topic-16-partitions-1kb.yaml b/workloads/backlog-1-topic-16-partitions-1kb.yaml index 11fce285d..d78180a07 100644 --- a/workloads/backlog-1-topic-16-partitions-1kb.yaml +++ b/workloads/backlog-1-topic-16-partitions-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: 1 producer / 1 consumers on 1 topic diff --git a/workloads/max-rate-1-topic-1-partition-1p-1c-100b.yaml b/workloads/max-rate-1-topic-1-partition-1p-1c-100b.yaml index 8ec41c45c..15ae9a5ea 100644 --- a/workloads/max-rate-1-topic-1-partition-1p-1c-100b.yaml +++ b/workloads/max-rate-1-topic-1-partition-1p-1c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-1-partition-1p-1c-100b diff --git a/workloads/max-rate-1-topic-1-partition-1p-1c-1kb.yaml b/workloads/max-rate-1-topic-1-partition-1p-1c-1kb.yaml index f135a5037..5f730b402 100644 --- a/workloads/max-rate-1-topic-1-partition-1p-1c-1kb.yaml +++ b/workloads/max-rate-1-topic-1-partition-1p-1c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-1-partition-1p-1c-1kb diff --git a/workloads/max-rate-1-topic-1-partition-1p-1c-64kb.yaml b/workloads/max-rate-1-topic-1-partition-1p-1c-64kb.yaml index 0438461fe..2b4f541ac 100644 --- a/workloads/max-rate-1-topic-1-partition-1p-1c-64kb.yaml +++ b/workloads/max-rate-1-topic-1-partition-1p-1c-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-1-partition-1p-1c-64kb diff --git a/workloads/max-rate-1-topic-1-partition-4p-1c-1kb.yaml b/workloads/max-rate-1-topic-1-partition-4p-1c-1kb.yaml index 166ece735..2e0dbc1a1 100644 --- a/workloads/max-rate-1-topic-1-partition-4p-1c-1kb.yaml +++ b/workloads/max-rate-1-topic-1-partition-4p-1c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-1-partition-4p-1c-1kb diff --git a/workloads/max-rate-1-topic-10-partitions-10p-10c-100b.yaml b/workloads/max-rate-1-topic-10-partitions-10p-10c-100b.yaml index b1b80f084..b12af94a3 100644 --- a/workloads/max-rate-1-topic-10-partitions-10p-10c-100b.yaml +++ b/workloads/max-rate-1-topic-10-partitions-10p-10c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-10-partitions-10p-10c-100b diff --git a/workloads/max-rate-1-topic-10-partitions-10p-10c-1kb.yaml b/workloads/max-rate-1-topic-10-partitions-10p-10c-1kb.yaml index 78393cc6e..fac203b03 100644 --- a/workloads/max-rate-1-topic-10-partitions-10p-10c-1kb.yaml +++ b/workloads/max-rate-1-topic-10-partitions-10p-10c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-10-partitions-10p-10c-1kb diff --git a/workloads/max-rate-1-topic-10-partitions-10p-10c-64kb.yaml b/workloads/max-rate-1-topic-10-partitions-10p-10c-64kb.yaml index 3e383ab82..546756561 100644 --- a/workloads/max-rate-1-topic-10-partitions-10p-10c-64kb.yaml +++ b/workloads/max-rate-1-topic-10-partitions-10p-10c-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-10-partitions-10p-10c-64kb diff --git a/workloads/max-rate-1-topic-10-partitions-1p-1c-100b.yaml b/workloads/max-rate-1-topic-10-partitions-1p-1c-100b.yaml index 8e3aa34e0..0130a8407 100644 --- a/workloads/max-rate-1-topic-10-partitions-1p-1c-100b.yaml +++ b/workloads/max-rate-1-topic-10-partitions-1p-1c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-100-partitions-1p-1c-100b diff --git a/workloads/max-rate-1-topic-10-partitions-1p-1c-1kb.yaml b/workloads/max-rate-1-topic-10-partitions-1p-1c-1kb.yaml index 59f59484a..f204d6874 100644 --- a/workloads/max-rate-1-topic-10-partitions-1p-1c-1kb.yaml +++ b/workloads/max-rate-1-topic-10-partitions-1p-1c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-100-partitions-1p-1c-1kb diff --git a/workloads/max-rate-1-topic-10-partitions-1p-1c-64kb.yaml b/workloads/max-rate-1-topic-10-partitions-1p-1c-64kb.yaml index e729b67ba..bc41c414b 100644 --- a/workloads/max-rate-1-topic-10-partitions-1p-1c-64kb.yaml +++ b/workloads/max-rate-1-topic-10-partitions-1p-1c-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-100-partitions-1p-1c-64kb diff --git a/workloads/max-rate-1-topic-100-partitions-100b.yaml b/workloads/max-rate-1-topic-100-partitions-100b.yaml index 4920c4f09..79e29a4f1 100644 --- a/workloads/max-rate-1-topic-100-partitions-100b.yaml +++ b/workloads/max-rate-1-topic-100-partitions-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Max rate 1 producer on 1 topic / 100 partition / 100 bytes diff --git a/workloads/max-rate-1-topic-100-partitions-100p-100c-100b.yaml b/workloads/max-rate-1-topic-100-partitions-100p-100c-100b.yaml index a027e5644..d7a390bf3 100644 --- a/workloads/max-rate-1-topic-100-partitions-100p-100c-100b.yaml +++ b/workloads/max-rate-1-topic-100-partitions-100p-100c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-100-partitions-100p-100c-100b diff --git a/workloads/max-rate-1-topic-100-partitions-100p-100c-1kb.yaml b/workloads/max-rate-1-topic-100-partitions-100p-100c-1kb.yaml index 3c2861341..d9bc894cd 100644 --- a/workloads/max-rate-1-topic-100-partitions-100p-100c-1kb.yaml +++ b/workloads/max-rate-1-topic-100-partitions-100p-100c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-100-partitions-100p-100c-1kb diff --git a/workloads/max-rate-1-topic-100-partitions-100p-100c-64kb.yaml b/workloads/max-rate-1-topic-100-partitions-100p-100c-64kb.yaml index f4dde89bc..f70b1341a 100644 --- a/workloads/max-rate-1-topic-100-partitions-100p-100c-64kb.yaml +++ b/workloads/max-rate-1-topic-100-partitions-100p-100c-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-100-partitions-100p-100c-64kb diff --git a/workloads/max-rate-1-topic-100-partitions-1kb.yaml b/workloads/max-rate-1-topic-100-partitions-1kb.yaml index 1e350d6d9..b176f6da0 100644 --- a/workloads/max-rate-1-topic-100-partitions-1kb.yaml +++ b/workloads/max-rate-1-topic-100-partitions-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Max rate 1 producer on 1 topic / 100 partition diff --git a/workloads/max-rate-1-topic-100-partitions-1p-1c-100b.yaml b/workloads/max-rate-1-topic-100-partitions-1p-1c-100b.yaml index b54129837..f4ee12735 100644 --- a/workloads/max-rate-1-topic-100-partitions-1p-1c-100b.yaml +++ b/workloads/max-rate-1-topic-100-partitions-1p-1c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-100-partitions-1p-1c-100b diff --git a/workloads/max-rate-1-topic-100-partitions-1p-1c-1kb.yaml b/workloads/max-rate-1-topic-100-partitions-1p-1c-1kb.yaml index 902fa91bb..9b3b19484 100644 --- a/workloads/max-rate-1-topic-100-partitions-1p-1c-1kb.yaml +++ b/workloads/max-rate-1-topic-100-partitions-1p-1c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-100-partitions-1p-1c-1kb diff --git a/workloads/max-rate-1-topic-16-partitions-100b.yaml b/workloads/max-rate-1-topic-16-partitions-100b.yaml index 31e36ac88..f5764c251 100644 --- a/workloads/max-rate-1-topic-16-partitions-100b.yaml +++ b/workloads/max-rate-1-topic-16-partitions-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Max rate 1 producer on 1 topic / 16 partitions diff --git a/workloads/max-rate-1-topic-16-partitions-1kb.yaml b/workloads/max-rate-1-topic-16-partitions-1kb.yaml index 83ece0c62..af5a78fbf 100644 --- a/workloads/max-rate-1-topic-16-partitions-1kb.yaml +++ b/workloads/max-rate-1-topic-16-partitions-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: Max rate 1 producer on 1 topic / 1 partition diff --git a/workloads/max-rate-1-topic-20-partitions-20p-20c-100b.yaml b/workloads/max-rate-1-topic-20-partitions-20p-20c-100b.yaml index 3fbb3053c..e5796028a 100644 --- a/workloads/max-rate-1-topic-20-partitions-20p-20c-100b.yaml +++ b/workloads/max-rate-1-topic-20-partitions-20p-20c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-20-partitions-20p-20c-100b diff --git a/workloads/max-rate-1-topic-20-partitions-20p-20c-1kb.yaml b/workloads/max-rate-1-topic-20-partitions-20p-20c-1kb.yaml index dc4b402fd..754422b74 100644 --- a/workloads/max-rate-1-topic-20-partitions-20p-20c-1kb.yaml +++ b/workloads/max-rate-1-topic-20-partitions-20p-20c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-20-partitions-20p-20c-1kb diff --git a/workloads/max-rate-1-topic-20-partitions-20p-20c-64kb.yaml b/workloads/max-rate-1-topic-20-partitions-20p-20c-64kb.yaml index 3592bb896..82792e2ab 100644 --- a/workloads/max-rate-1-topic-20-partitions-20p-20c-64kb.yaml +++ b/workloads/max-rate-1-topic-20-partitions-20p-20c-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-20-partitions-20p-20c-64kb diff --git a/workloads/max-rate-1-topic-30-partitions-30p-30c-100b.yaml b/workloads/max-rate-1-topic-30-partitions-30p-30c-100b.yaml index 3ee40d0bf..81d7744b8 100644 --- a/workloads/max-rate-1-topic-30-partitions-30p-30c-100b.yaml +++ b/workloads/max-rate-1-topic-30-partitions-30p-30c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-30-partitions-30p-30c-100b diff --git a/workloads/max-rate-1-topic-30-partitions-30p-30c-1kb.yaml b/workloads/max-rate-1-topic-30-partitions-30p-30c-1kb.yaml index 79e036626..f4e46225a 100644 --- a/workloads/max-rate-1-topic-30-partitions-30p-30c-1kb.yaml +++ b/workloads/max-rate-1-topic-30-partitions-30p-30c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-30-partitions-30p-30c-1kb diff --git a/workloads/max-rate-1-topic-40-partitions-40p-40c-100b.yaml b/workloads/max-rate-1-topic-40-partitions-40p-40c-100b.yaml index cc647d349..ce99582af 100644 --- a/workloads/max-rate-1-topic-40-partitions-40p-40c-100b.yaml +++ b/workloads/max-rate-1-topic-40-partitions-40p-40c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-40-partitions-40p-40c-100b diff --git a/workloads/max-rate-1-topic-40-partitions-40p-40c-1kb.yaml b/workloads/max-rate-1-topic-40-partitions-40p-40c-1kb.yaml index 86a7cecb1..c31ab6882 100644 --- a/workloads/max-rate-1-topic-40-partitions-40p-40c-1kb.yaml +++ b/workloads/max-rate-1-topic-40-partitions-40p-40c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-40-partitions-40p-40c-1kb diff --git a/workloads/max-rate-1-topic-40-partitions-40p-40c-64kb.yaml b/workloads/max-rate-1-topic-40-partitions-40p-40c-64kb.yaml index 42fea3c31..215d7b9cd 100644 --- a/workloads/max-rate-1-topic-40-partitions-40p-40c-64kb.yaml +++ b/workloads/max-rate-1-topic-40-partitions-40p-40c-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-40-partitions-40p-40c-64kb diff --git a/workloads/max-rate-1-topic-50-partitions-50p-50c-100b.yaml b/workloads/max-rate-1-topic-50-partitions-50p-50c-100b.yaml index 16fb69222..1a84a34b4 100644 --- a/workloads/max-rate-1-topic-50-partitions-50p-50c-100b.yaml +++ b/workloads/max-rate-1-topic-50-partitions-50p-50c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-50-partitions-50p-50c-100b diff --git a/workloads/max-rate-1-topic-50-partitions-50p-50c-1kb.yaml b/workloads/max-rate-1-topic-50-partitions-50p-50c-1kb.yaml index 332c3ca9d..76c63c7bb 100644 --- a/workloads/max-rate-1-topic-50-partitions-50p-50c-1kb.yaml +++ b/workloads/max-rate-1-topic-50-partitions-50p-50c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-50-partitions-50p-50c-1kb diff --git a/workloads/max-rate-1-topic-60-partitions-60p-60c-100b.yaml b/workloads/max-rate-1-topic-60-partitions-60p-60c-100b.yaml index 0b1530a67..40ed6a9db 100644 --- a/workloads/max-rate-1-topic-60-partitions-60p-60c-100b.yaml +++ b/workloads/max-rate-1-topic-60-partitions-60p-60c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-60-partitions-60p-60c-100b diff --git a/workloads/max-rate-1-topic-60-partitions-60p-60c-1kb.yaml b/workloads/max-rate-1-topic-60-partitions-60p-60c-1kb.yaml index f26063443..4cac48e95 100644 --- a/workloads/max-rate-1-topic-60-partitions-60p-60c-1kb.yaml +++ b/workloads/max-rate-1-topic-60-partitions-60p-60c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-60-partitions-60p-60c-1kb diff --git a/workloads/max-rate-1-topic-60-partitions-60p-60c-64kb.yaml b/workloads/max-rate-1-topic-60-partitions-60p-60c-64kb.yaml index ec37f7252..2a37f1324 100644 --- a/workloads/max-rate-1-topic-60-partitions-60p-60c-64kb.yaml +++ b/workloads/max-rate-1-topic-60-partitions-60p-60c-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-60-partitions-60p-60c-64kb diff --git a/workloads/max-rate-1-topic-70-partitions-70p-70c-100b.yaml b/workloads/max-rate-1-topic-70-partitions-70p-70c-100b.yaml index d96f09fb2..296d48117 100644 --- a/workloads/max-rate-1-topic-70-partitions-70p-70c-100b.yaml +++ b/workloads/max-rate-1-topic-70-partitions-70p-70c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-70-partitions-70p-70c-100b diff --git a/workloads/max-rate-1-topic-70-partitions-70p-70c-1kb.yaml b/workloads/max-rate-1-topic-70-partitions-70p-70c-1kb.yaml index 4870820f8..220abae74 100644 --- a/workloads/max-rate-1-topic-70-partitions-70p-70c-1kb.yaml +++ b/workloads/max-rate-1-topic-70-partitions-70p-70c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-70-partitions-70p-70c-1kb diff --git a/workloads/max-rate-1-topic-80-partitions-80p-80c-100b.yaml b/workloads/max-rate-1-topic-80-partitions-80p-80c-100b.yaml index 585091a02..7a5288c41 100644 --- a/workloads/max-rate-1-topic-80-partitions-80p-80c-100b.yaml +++ b/workloads/max-rate-1-topic-80-partitions-80p-80c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-80-partitions-80p-80c-100b diff --git a/workloads/max-rate-1-topic-80-partitions-80p-80c-1kb.yaml b/workloads/max-rate-1-topic-80-partitions-80p-80c-1kb.yaml index 497245402..640bdc0d6 100644 --- a/workloads/max-rate-1-topic-80-partitions-80p-80c-1kb.yaml +++ b/workloads/max-rate-1-topic-80-partitions-80p-80c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-80-partitions-80p-80c-1kb diff --git a/workloads/max-rate-1-topic-80-partitions-80p-80c-64kb.yaml b/workloads/max-rate-1-topic-80-partitions-80p-80c-64kb.yaml index dba8495bd..bb176749c 100644 --- a/workloads/max-rate-1-topic-80-partitions-80p-80c-64kb.yaml +++ b/workloads/max-rate-1-topic-80-partitions-80p-80c-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-80-partitions-80p-80c-64kb diff --git a/workloads/max-rate-1-topic-90-partitions-90p-90c-100b.yaml b/workloads/max-rate-1-topic-90-partitions-90p-90c-100b.yaml index ef6f31ee4..c3b3a2dbd 100644 --- a/workloads/max-rate-1-topic-90-partitions-90p-90c-100b.yaml +++ b/workloads/max-rate-1-topic-90-partitions-90p-90c-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-90-partitions-90p-90c-100b diff --git a/workloads/max-rate-1-topic-90-partitions-90p-90c-1kb.yaml b/workloads/max-rate-1-topic-90-partitions-90p-90c-1kb.yaml index 424fa7f87..cd49480b2 100644 --- a/workloads/max-rate-1-topic-90-partitions-90p-90c-1kb.yaml +++ b/workloads/max-rate-1-topic-90-partitions-90p-90c-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-1-topic-90-partitions-90p-90c-1kb diff --git a/workloads/max-rate-10-topics-1-partition-100b.yaml b/workloads/max-rate-10-topics-1-partition-100b.yaml index dee18356c..c8891dcf1 100644 --- a/workloads/max-rate-10-topics-1-partition-100b.yaml +++ b/workloads/max-rate-10-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-10-topics-1-partition-100b diff --git a/workloads/max-rate-10-topics-1-partition-1kb.yaml b/workloads/max-rate-10-topics-1-partition-1kb.yaml index 4f3b905d6..6add8303a 100644 --- a/workloads/max-rate-10-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-10-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-10-topics-1-partition-1kb diff --git a/workloads/max-rate-10-topics-1-partition-64kb.yaml b/workloads/max-rate-10-topics-1-partition-64kb.yaml index 5dfbd0b90..092b92b50 100644 --- a/workloads/max-rate-10-topics-1-partition-64kb.yaml +++ b/workloads/max-rate-10-topics-1-partition-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-10-topics-1-partition-64kb diff --git a/workloads/max-rate-100-topics-1-partition-100b.yaml b/workloads/max-rate-100-topics-1-partition-100b.yaml index 2db091839..9c89d3f06 100644 --- a/workloads/max-rate-100-topics-1-partition-100b.yaml +++ b/workloads/max-rate-100-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-100-topics-1-partition-100b diff --git a/workloads/max-rate-100-topics-1-partition-1kb.yaml b/workloads/max-rate-100-topics-1-partition-1kb.yaml index f47884325..e236607c3 100644 --- a/workloads/max-rate-100-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-100-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-100-topics-1-partition-1kb diff --git a/workloads/max-rate-100-topics-1-partition-64kb.yaml b/workloads/max-rate-100-topics-1-partition-64kb.yaml index 63f868883..510c7dee5 100644 --- a/workloads/max-rate-100-topics-1-partition-64kb.yaml +++ b/workloads/max-rate-100-topics-1-partition-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-100-topics-1-partition-64kb diff --git a/workloads/max-rate-20-topics-1-partition-100b.yaml b/workloads/max-rate-20-topics-1-partition-100b.yaml index f77913e73..92d6912a0 100644 --- a/workloads/max-rate-20-topics-1-partition-100b.yaml +++ b/workloads/max-rate-20-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-20-topics-1-partition-100b diff --git a/workloads/max-rate-20-topics-1-partition-1kb.yaml b/workloads/max-rate-20-topics-1-partition-1kb.yaml index 18f9a9f11..4aa53b53b 100644 --- a/workloads/max-rate-20-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-20-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-20-topics-1-partition-1kb diff --git a/workloads/max-rate-20-topics-1-partition-64kb.yaml b/workloads/max-rate-20-topics-1-partition-64kb.yaml index c858dfeda..2e563175a 100644 --- a/workloads/max-rate-20-topics-1-partition-64kb.yaml +++ b/workloads/max-rate-20-topics-1-partition-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-20-topics-1-partition-64kb diff --git a/workloads/max-rate-30-topics-1-partition-100b.yaml b/workloads/max-rate-30-topics-1-partition-100b.yaml index b95a6dcef..f02fb7bed 100644 --- a/workloads/max-rate-30-topics-1-partition-100b.yaml +++ b/workloads/max-rate-30-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-30-topics-1-partition-100b diff --git a/workloads/max-rate-30-topics-1-partition-1kb.yaml b/workloads/max-rate-30-topics-1-partition-1kb.yaml index 87d84f57a..0aab511eb 100644 --- a/workloads/max-rate-30-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-30-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-30-topics-1-partition-1kb diff --git a/workloads/max-rate-40-topics-1-partition-100b.yaml b/workloads/max-rate-40-topics-1-partition-100b.yaml index 79e56614e..55c449ac3 100644 --- a/workloads/max-rate-40-topics-1-partition-100b.yaml +++ b/workloads/max-rate-40-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-40-topics-1-partition-100b diff --git a/workloads/max-rate-40-topics-1-partition-1kb.yaml b/workloads/max-rate-40-topics-1-partition-1kb.yaml index 44cbbbb48..a247d70a2 100644 --- a/workloads/max-rate-40-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-40-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-40-topics-1-partition-1kb diff --git a/workloads/max-rate-40-topics-1-partition-64kb.yaml b/workloads/max-rate-40-topics-1-partition-64kb.yaml index 9db67a0b1..dd8ec9724 100644 --- a/workloads/max-rate-40-topics-1-partition-64kb.yaml +++ b/workloads/max-rate-40-topics-1-partition-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-40-topics-1-partition-64kb diff --git a/workloads/max-rate-50-topics-1-partition-100b.yaml b/workloads/max-rate-50-topics-1-partition-100b.yaml index 9b9df80fb..0980d52f9 100644 --- a/workloads/max-rate-50-topics-1-partition-100b.yaml +++ b/workloads/max-rate-50-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-50-topics-1-partition-100b diff --git a/workloads/max-rate-50-topics-1-partition-1kb.yaml b/workloads/max-rate-50-topics-1-partition-1kb.yaml index de3d02d68..43de6a3f5 100644 --- a/workloads/max-rate-50-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-50-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-50-topics-1-partition-1kb diff --git a/workloads/max-rate-60-topics-1-partition-100b.yaml b/workloads/max-rate-60-topics-1-partition-100b.yaml index 0bd4ded1d..db87c08ad 100644 --- a/workloads/max-rate-60-topics-1-partition-100b.yaml +++ b/workloads/max-rate-60-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-60-topics-1-partition-100b diff --git a/workloads/max-rate-60-topics-1-partition-1kb.yaml b/workloads/max-rate-60-topics-1-partition-1kb.yaml index d22b2622d..7066fc38c 100644 --- a/workloads/max-rate-60-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-60-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-60-topics-1-partition-1kb diff --git a/workloads/max-rate-60-topics-1-partition-64kb.yaml b/workloads/max-rate-60-topics-1-partition-64kb.yaml index 8344e2909..02c6af28c 100644 --- a/workloads/max-rate-60-topics-1-partition-64kb.yaml +++ b/workloads/max-rate-60-topics-1-partition-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-60-topics-1-partition-64kb diff --git a/workloads/max-rate-70-topics-1-partition-100b.yaml b/workloads/max-rate-70-topics-1-partition-100b.yaml index 754523ac2..a1689162e 100644 --- a/workloads/max-rate-70-topics-1-partition-100b.yaml +++ b/workloads/max-rate-70-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-70-topics-1-partition-100b diff --git a/workloads/max-rate-70-topics-1-partition-1kb.yaml b/workloads/max-rate-70-topics-1-partition-1kb.yaml index 0deb78b30..d51208c6a 100644 --- a/workloads/max-rate-70-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-70-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-70-topics-1-partition-1kb diff --git a/workloads/max-rate-80-topics-1-partition-100b.yaml b/workloads/max-rate-80-topics-1-partition-100b.yaml index c59fe85a7..9de875843 100644 --- a/workloads/max-rate-80-topics-1-partition-100b.yaml +++ b/workloads/max-rate-80-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-80-topics-1-partition-100b diff --git a/workloads/max-rate-80-topics-1-partition-1kb.yaml b/workloads/max-rate-80-topics-1-partition-1kb.yaml index 77123de89..1ff47d2e7 100644 --- a/workloads/max-rate-80-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-80-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-80-topics-1-partition-1kb diff --git a/workloads/max-rate-80-topics-1-partition-64kb.yaml b/workloads/max-rate-80-topics-1-partition-64kb.yaml index 8a19750b7..a79d7ea6d 100644 --- a/workloads/max-rate-80-topics-1-partition-64kb.yaml +++ b/workloads/max-rate-80-topics-1-partition-64kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-80-topics-1-partition-64kb diff --git a/workloads/max-rate-90-topics-1-partition-100b.yaml b/workloads/max-rate-90-topics-1-partition-100b.yaml index 68b95cd7f..5cef920f0 100644 --- a/workloads/max-rate-90-topics-1-partition-100b.yaml +++ b/workloads/max-rate-90-topics-1-partition-100b.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-90-topics-1-partition-100b diff --git a/workloads/max-rate-90-topics-1-partition-1kb.yaml b/workloads/max-rate-90-topics-1-partition-1kb.yaml index 360f2cf0e..3d123fd18 100644 --- a/workloads/max-rate-90-topics-1-partition-1kb.yaml +++ b/workloads/max-rate-90-topics-1-partition-1kb.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # name: max-rate-90-topics-1-partition-1kb diff --git a/workloads/simple-workload.yaml b/workloads/simple-workload.yaml index f3f17fbec..bf3a5ea97 100644 --- a/workloads/simple-workload.yaml +++ b/workloads/simple-workload.yaml @@ -1,20 +1,15 @@ # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # From 214e33e10c1b5eb944e90cbed6465ceb4d7797d4 Mon Sep 17 00:00:00 2001 From: von gosling Date: Fri, 8 Apr 2022 10:30:30 +0800 Subject: [PATCH 016/103] Revert "[Pravega Driver] [Draft] Update dependencies for Pravega 0.12 (#259)" (#264) This reverts commit 800e1afd5863baa4c9e932b94a2f2e69fe480a23. --- driver-pravega/pom.xml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/driver-pravega/pom.xml b/driver-pravega/pom.xml index 62fac1092..6989533a8 100644 --- a/driver-pravega/pom.xml +++ b/driver-pravega/pom.xml @@ -23,9 +23,9 @@ driver-pravega - - - 0.12.0 + + + 0.10.1 @@ -61,14 +61,7 @@ io.grpc grpc-all - - 1.39.0 - - - - io.netty - netty-tcnative-boringssl-static - 2.0.39.Final + 1.36.0 From 2d5e56879598bdc97b1fa563474cb37b2e1594b2 Mon Sep 17 00:00:00 2001 From: zhangjidi2016 Date: Tue, 12 Apr 2022 09:59:35 +0800 Subject: [PATCH 017/103] [ISSUE #265]Fix com/fasterxml/jackson/annotation/JsonKey: NoClassDefFoundError (#266) Co-authored-by: zhangjidi2016 --- benchmark-framework/pom.xml | 9 +++++---- pom.xml | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index 9e4b75543..db3dd6ae3 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -25,6 +25,7 @@ 2.17.1 + 2.13.2 @@ -54,23 +55,23 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-base - 2.9.3 + ${jackson.version} com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - 2.9.3 + ${jackson.version} com.fasterxml.jackson.core jackson-annotations - 2.9.3 + ${jackson.version} com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.9.3 + ${jackson.version} diff --git a/pom.xml b/pom.xml index a5e1c9e4c..3ad7f0eac 100644 --- a/pom.xml +++ b/pom.xml @@ -55,6 +55,7 @@ UTF-8 UTF-8 4.14.0 + 2.13.2 @@ -103,13 +104,13 @@ com.fasterxml.jackson.core jackson-databind - 2.12.6.1 + ${jackson.version} com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.9.3 + ${jackson.version} io.netty From c1200d632098d6da35b328858b8d514b9c105a40 Mon Sep 17 00:00:00 2001 From: zhangjidi2016 Date: Sun, 1 May 2022 11:55:49 +0800 Subject: [PATCH 018/103] [ISSUE #268]rocketmq benchmark support acl (#269) Co-authored-by: zhangjidi --- driver-rocketmq/pom.xml | 19 ++++++++- driver-rocketmq/rocketmq.yaml | 2 + .../rocketmq/RocketMQBenchmarkDriver.java | 41 +++++++++++++++---- .../rocketmq/client/RocketMQClientConfig.java | 2 + 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/driver-rocketmq/pom.xml b/driver-rocketmq/pom.xml index cae20cadc..dc388dd56 100644 --- a/driver-rocketmq/pom.xml +++ b/driver-rocketmq/pom.xml @@ -23,6 +23,11 @@ driver-rocketmq + + 4.9.3 + 29.0-jre + + ${project.groupId} @@ -30,15 +35,25 @@ ${project.version} + + org.apache.rocketmq + rocketmq-acl + ${rocketmq.version} + org.apache.rocketmq rocketmq-client - 4.3.0 + ${rocketmq.version} org.apache.rocketmq rocketmq-tools - 4.3.0 + ${rocketmq.version} + + + com.google.guava + guava + ${guava.version} diff --git a/driver-rocketmq/rocketmq.yaml b/driver-rocketmq/rocketmq.yaml index e075120ee..b5aec62ee 100644 --- a/driver-rocketmq/rocketmq.yaml +++ b/driver-rocketmq/rocketmq.yaml @@ -19,3 +19,5 @@ driverClass: io.openmessaging.benchmark.driver.rocketmq.RocketMQBenchmarkDriver clusterName: DefaultCluster namesrvAddr: 127.0.0.1:9876 vipChannelEnabled: false +accessKey: +secretKey: \ No newline at end of file diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java index 10fd68d06..8ac7c1f72 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java @@ -29,13 +29,18 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.SessionCredentials; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.CommandUtil; import org.slf4j.Logger; @@ -45,11 +50,17 @@ public class RocketMQBenchmarkDriver implements BenchmarkDriver { private DefaultMQAdminExt rmqAdmin; private RocketMQClientConfig rmqClientConfig; DefaultMQProducer rmqProducer; + private RPCHook rpcHook; + @Override public void initialize(final File configurationFile, final StatsLogger statsLogger) throws IOException { this.rmqClientConfig = readConfig(configurationFile); - - this.rmqAdmin = new DefaultMQAdminExt(); + if (isAclEnabled()) { + rpcHook = new AclClientRPCHook(new SessionCredentials(this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey)); + this.rmqAdmin = new DefaultMQAdminExt(rpcHook); + } else { + this.rmqAdmin = new DefaultMQAdminExt(); + } this.rmqAdmin.setNamesrvAddr(this.rmqClientConfig.namesrvAddr); this.rmqAdmin.setInstanceName("AdminInstance-" + getRandomString()); try { @@ -92,16 +103,20 @@ public CompletableFuture createTopic(final String topic, final int partiti @Override public CompletableFuture createProducer(final String topic) { if (rmqProducer == null) { - rmqProducer = new DefaultMQProducer("ProducerGroup_" + getRandomString()); + if (isAclEnabled()) { + rmqProducer = new DefaultMQProducer("ProducerGroup_" + getRandomString(), rpcHook); + } else { + rmqProducer = new DefaultMQProducer("ProducerGroup_" + getRandomString()); + } rmqProducer.setNamesrvAddr(this.rmqClientConfig.namesrvAddr); rmqProducer.setInstanceName("ProducerInstance" + getRandomString()); - if(null != this.rmqClientConfig.vipChannelEnabled){ + if (null != this.rmqClientConfig.vipChannelEnabled) { rmqProducer.setVipChannelEnabled(this.rmqClientConfig.vipChannelEnabled); } - if(null != this.rmqClientConfig.maxMessageSize){ + if (null != this.rmqClientConfig.maxMessageSize) { rmqProducer.setMaxMessageSize(this.rmqClientConfig.maxMessageSize); } - if(null != this.rmqClientConfig.compressMsgBodyOverHowmuch){ + if (null != this.rmqClientConfig.compressMsgBodyOverHowmuch) { rmqProducer.setCompressMsgBodyOverHowmuch(this.rmqClientConfig.compressMsgBodyOverHowmuch); } try { @@ -117,10 +132,15 @@ public CompletableFuture createProducer(final String topic) { @Override public CompletableFuture createConsumer(final String topic, final String subscriptionName, final ConsumerCallback consumerCallback) { - DefaultMQPushConsumer rmqConsumer = new DefaultMQPushConsumer(subscriptionName); + DefaultMQPushConsumer rmqConsumer; + if (isAclEnabled()) { + rmqConsumer = new DefaultMQPushConsumer(subscriptionName, rpcHook, new AllocateMessageQueueAveragely()); + } else { + rmqConsumer = new DefaultMQPushConsumer(subscriptionName); + } rmqConsumer.setNamesrvAddr(this.rmqClientConfig.namesrvAddr); rmqConsumer.setInstanceName("ConsumerInstance" + getRandomString()); - if(null != this.rmqClientConfig.vipChannelEnabled){ + if (null != this.rmqClientConfig.vipChannelEnabled) { rmqConsumer.setVipChannelEnabled(this.rmqClientConfig.vipChannelEnabled); } try { @@ -139,6 +159,11 @@ public CompletableFuture createConsumer(final String topic, f return CompletableFuture.completedFuture(new RocketMQBenchmarkConsumer(rmqConsumer)); } + public boolean isAclEnabled() { + return !(StringUtils.isAnyBlank(this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey) || + StringUtils.isAnyEmpty(this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey)); + } + @Override public void close() throws Exception { if (this.rmqProducer != null) { diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java index f9de02d7e..840090452 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java @@ -19,4 +19,6 @@ public class RocketMQClientConfig { public Boolean vipChannelEnabled; public Integer maxMessageSize; public Integer compressMsgBodyOverHowmuch; + public String accessKey; + public String secretKey; } From 4a02411992f5089c466e3e73ccbad22cf6cf0f1b Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Jul 2022 10:28:47 -0700 Subject: [PATCH 019/103] Switch to RHEL-8 since RHEL-7.4 doesn't work anymore on EC2 --- driver-pulsar/deploy/ssd/deploy.yaml | 4 ++-- driver-pulsar/deploy/ssd/terraform.tfvars | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index db28af30e..f3f9f1e13 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -136,8 +136,8 @@ state: latest pkg: - wget - - java-11-openjdk - - java-11-openjdk-devel + - java-17-openjdk + - java-17-openjdk-devel - sysstat - vim - chrony diff --git a/driver-pulsar/deploy/ssd/terraform.tfvars b/driver-pulsar/deploy/ssd/terraform.tfvars index 62b012179..f36a8621b 100644 --- a/driver-pulsar/deploy/ssd/terraform.tfvars +++ b/driver-pulsar/deploy/ssd/terraform.tfvars @@ -1,7 +1,7 @@ public_key_path = "~/.ssh/pulsar_aws.pub" region = "us-west-2" az = "us-west-2a" -ami = "ami-9fa343e7" // RHEL-7.4 +ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { "pulsar" = "i3en.6xlarge" From 7ec181716f88060b6e39b57b0465ac04ed8524fc Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Jul 2022 13:30:16 -0700 Subject: [PATCH 020/103] Simplified JVM GC options --- driver-pulsar/deploy/ssd/templates/pulsar_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/driver-pulsar/deploy/ssd/templates/pulsar_env.sh b/driver-pulsar/deploy/ssd/templates/pulsar_env.sh index deb69f719..b33df2ee6 100644 --- a/driver-pulsar/deploy/ssd/templates/pulsar_env.sh +++ b/driver-pulsar/deploy/ssd/templates/pulsar_env.sh @@ -40,8 +40,8 @@ PULSAR_MEM=" -Xms{{ max_heap_memory }} -Xmx{{ max_heap_memory }} -XX:MaxDirectMemorySize={{ max_direct_memory }}" # Garbage collection options -PULSAR_GC=" -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB" -PULSAR_GC="${PULSAR_GC} -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking" +PULSAR_GC=" -XX:+UseZGC" +PULSAR_GC="${PULSAR_GC} -XX:+PerfDisableSharedMem" # Extra options to be passed to the jvm PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" From 48c31dbdcf2b411f96f7010be2cf0a684697b973 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Jul 2022 14:03:03 -0700 Subject: [PATCH 021/103] Disabling protocol handlers in ansible --- driver-pulsar/deploy/ssd/deploy.yaml | 98 +++++++++---------- .../deploy/ssd/templates/broker.conf | 7 +- 2 files changed, 55 insertions(+), 50 deletions(-) diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index f3f9f1e13..35c6bd30c 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -161,29 +161,29 @@ echo 'LANG=en_US.utf-8 LC_ALL=en_US.utf-8' > /etc/environment -- name: Protocol Handlers installation - hosts: pulsar - connection: ssh - become: true - tasks: - - file: path=/opt/pulsar/protocols state=absent - - file: path=/opt/pulsar/protocols state=directory - - name: Upload local protocol handlers - copy: - src: "{{ item.local_path }}" - dest: /opt/pulsar/protocols - loop: "{{ protocol_handlers }}" - when: - - protocol_handlers is defined - - item.local_path is defined - - name: Download protocol handlers - get_url: - url: "{{ item.url }}" - dest: /opt/pulsar/protocols - loop: "{{ protocol_handlers }}" - when: - - protocol_handlers is defined - - item.local_path is not defined +#- name: Protocol Handlers installation +# hosts: pulsar +# connection: ssh +# become: true +# tasks: +# - file: path=/opt/pulsar/protocols state=absent +# - file: path=/opt/pulsar/protocols state=directory +# - name: Upload local protocol handlers +# copy: +# src: "{{ item.local_path }}" +# dest: /opt/pulsar/protocols +# loop: "{{ protocol_handlers }}" +# when: +# - protocol_handlers is defined +# - item.local_path is defined +# - name: Download protocol handlers +# get_url: +# url: "{{ item.url }}" +# dest: /opt/pulsar/protocols +# loop: "{{ protocol_handlers }}" +# when: +# - protocol_handlers is defined +# - item.local_path is not defined - name: ZooKeeper setup hosts: zookeeper @@ -263,32 +263,32 @@ - template: src: "templates/pulsar.service" dest: "/etc/systemd/system/pulsar.service" - - template: - src: "templates/{{ item.conf }}" - dest: "/opt/pulsar/conf/{{ item.conf }}" - loop: "{{ protocol_handlers }}" - when: protocol_handlers is defined - - set_fact: - protocols: "{{ protocols | default([]) + [ item.protocol ]}}" - loop: "{{ protocol_handlers }}" - when: protocol_handlers is defined - - name: Enable protocol handlers - lineinfile: - path: /opt/pulsar/conf/broker.conf - line: "messagingProtocols={{ protocols | join(',') }}" - when: protocols is defined - - name: Read configurations of all protocol handlers - shell: | - grep -v "^#" "{{ '/opt/pulsar/conf/' + item.conf }}" - loop: "{{ protocol_handlers }}" - when: protocol_handlers is defined - register: conf_files - - name: Update broker.conf for protocol handlers - lineinfile: - path: /opt/pulsar/conf/broker.conf - line: "{{ item.stdout_lines | join('\n') }}" - loop: "{{ conf_files.results }}" - when: conf_files is defined +# - template: +# src: "templates/{{ item.conf }}" +# dest: "/opt/pulsar/conf/{{ item.conf }}" +# loop: "{{ protocol_handlers }}" +# when: protocol_handlers is defined +# - set_fact: +# protocols: "{{ protocols | default([]) + [ item.protocol ]}}" +# loop: "{{ protocol_handlers }}" +# when: protocol_handlers is defined +# - name: Enable protocol handlers +# lineinfile: +# path: /opt/pulsar/conf/broker.conf +# line: "messagingProtocols={{ protocols | join(',') }}" +# when: protocols is defined +# - name: Read configurations of all protocol handlers +# shell: | +# grep -v "^#" "{{ '/opt/pulsar/conf/' + item.conf }}" +# loop: "{{ protocol_handlers }}" +# when: protocol_handlers is defined +# register: conf_files +# - name: Update broker.conf for protocol handlers +# lineinfile: +# path: /opt/pulsar/conf/broker.conf +# line: "{{ item.stdout_lines | join('\n') }}" +# loop: "{{ conf_files.results }}" +# when: conf_files is defined - systemd: state: restarted daemon_reload: yes diff --git a/driver-pulsar/deploy/ssd/templates/broker.conf b/driver-pulsar/deploy/ssd/templates/broker.conf index 6bb8685c4..53137c2fa 100644 --- a/driver-pulsar/deploy/ssd/templates/broker.conf +++ b/driver-pulsar/deploy/ssd/templates/broker.conf @@ -34,4 +34,9 @@ defaultNumberOfNamespaceBundles=64 managedLedgerNewEntriesCheckDelayInMillis=0 -bookkeeperNumberOfChannelsPerBookie=64 \ No newline at end of file +bookkeeperNumberOfChannelsPerBookie=64 + +managedLedgerCacheEvictionFrequency=0.2 + +# Increasing from default to avoid throttling lookups +maxConcurrentLookupRequest=500000 \ No newline at end of file From 0aa74e34d1f5c462645bdc8dc20ec0d6c6c4d7a8 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Jul 2022 14:35:29 -0700 Subject: [PATCH 022/103] Upgrade to Pulsar 2.10.1 --- driver-pulsar/deploy/ssd/deploy.yaml | 2 +- driver-pulsar/deploy/ssd/extra_vars.yaml | 2 +- driver-pulsar/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index 35c6bd30c..693d86a5d 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -23,7 +23,7 @@ serviceUrl: "pulsar://{{ hostvars[groups['pulsar'][0]].private_ip }}:6650/" httpUrl: "http://{{ hostvars[groups['pulsar'][0]].private_ip }}:8080/" - set_fact: - pulsar_version: "{{ pulsar_version | default('2.9.1') }}" + pulsar_version: "{{ pulsar_version | default('2.10.1') }}" node_exporter_version: "{{ node_exporter_version | default('1.2.2') }}" prometheus_version: "{{ prometheus_version | default('2.31.1') }}" disk_dev: "{{ disk_dev | default(['/dev/nvme1n1', '/dev/nvme2n1']) }}" diff --git a/driver-pulsar/deploy/ssd/extra_vars.yaml b/driver-pulsar/deploy/ssd/extra_vars.yaml index ee09257ad..3d7385a01 100644 --- a/driver-pulsar/deploy/ssd/extra_vars.yaml +++ b/driver-pulsar/deploy/ssd/extra_vars.yaml @@ -12,7 +12,7 @@ # limitations under the License. # -pulsar_version: 2.9.1 +pulsar_version: 2.10.1 node_exporter_version: 1.2.2 prometheus_version: 2.31.1 diff --git a/driver-pulsar/pom.xml b/driver-pulsar/pom.xml index 2b7f10c0c..5606703d8 100644 --- a/driver-pulsar/pom.xml +++ b/driver-pulsar/pom.xml @@ -23,7 +23,7 @@ driver-pulsar - 2.9.1 + 2.10.1 From 72778d8a27405c14756f28c7ea02357eb7201b01 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Jul 2022 21:46:32 -0700 Subject: [PATCH 023/103] [Pulsar] Specify the NIC speed when deploying brokers --- driver-pulsar/deploy/ssd/templates/broker.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/driver-pulsar/deploy/ssd/templates/broker.conf b/driver-pulsar/deploy/ssd/templates/broker.conf index 53137c2fa..6397826df 100644 --- a/driver-pulsar/deploy/ssd/templates/broker.conf +++ b/driver-pulsar/deploy/ssd/templates/broker.conf @@ -39,4 +39,7 @@ bookkeeperNumberOfChannelsPerBookie=64 managedLedgerCacheEvictionFrequency=0.2 # Increasing from default to avoid throttling lookups -maxConcurrentLookupRequest=500000 \ No newline at end of file +maxConcurrentLookupRequest=500000 + +loadBalancerOverrideBrokerNicSpeedGbps=25 + From 3241915311cefa8965e9d40827136d91e7281dee Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Jul 2022 21:47:02 -0700 Subject: [PATCH 024/103] Fixed benchmark worker config to work with Java 17 --- driver-pulsar/deploy/ssd/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index 693d86a5d..60c101523 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -390,7 +390,7 @@ lineinfile: dest: /opt/benchmark/bin/benchmark-worker regexp: '^JVM_MEM=' - line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB"' + line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UseZGC"' - name: Configure memory lineinfile: dest: /opt/benchmark/bin/benchmark From 5b8209f6e75fa938396f6538842bb991c93c74a7 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Jul 2022 22:42:18 -0700 Subject: [PATCH 025/103] Use bigger VMs for ZooKeeper deployment --- driver-pulsar/deploy/ssd/deploy.yaml | 4 ++-- driver-pulsar/deploy/ssd/terraform.tfvars | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index 60c101523..30d8c3496 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -192,8 +192,8 @@ tasks: - set_fact: zid: "{{ groups['zookeeper'].index(inventory_hostname) }}" - max_heap_memory: "{{ zookeeper_max_heap_memory | default('512m') }}" - max_direct_memory: "{{ zookeeper_max_direct_memory | default('512m') }}" + max_heap_memory: "{{ zookeeper_max_heap_memory | default('10G') }}" + max_direct_memory: "{{ zookeeper_max_direct_memory | default('2G') }}" - file: path: "/opt/pulsar/{{ item }}" state: directory diff --git a/driver-pulsar/deploy/ssd/terraform.tfvars b/driver-pulsar/deploy/ssd/terraform.tfvars index f36a8621b..84503c1ec 100644 --- a/driver-pulsar/deploy/ssd/terraform.tfvars +++ b/driver-pulsar/deploy/ssd/terraform.tfvars @@ -5,7 +5,7 @@ ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { "pulsar" = "i3en.6xlarge" - "zookeeper" = "t2.small" + "zookeeper" = "t2.xlarge" "client" = "m5n.8xlarge" "prometheus" = "t2.large" } From 7a16bc5c5cd1ebebb1f630ac5697c3ca88ef02ff Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 3 Jul 2022 23:09:01 -0700 Subject: [PATCH 026/103] Fixed BK memory settings --- driver-pulsar/deploy/ssd/deploy.yaml | 7 +- driver-pulsar/deploy/ssd/extra_vars.yaml | 8 +- driver-pulsar/deploy/ssd/templates/bkenv.sh | 96 +++++++++++++++++++++ 3 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 driver-pulsar/deploy/ssd/templates/bkenv.sh diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index 30d8c3496..2e53000f8 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -151,11 +151,14 @@ dest: /opt/pulsar extra_opts: ["--strip-components=1"] - set_fact: - max_heap_memory: "{{ pulsar_max_heap_memory | default('16g') }}" - max_direct_memory: "{{ pulsar_max_direct_memory | default('48g') }}" + max_heap_memory: "{{ pulsar_max_heap_memory | default('32g') }}" + max_direct_memory: "{{ pulsar_max_direct_memory | default('32g') }}" - template: src: "templates/pulsar_env.sh" dest: "/opt/pulsar/conf/pulsar_env.sh" + - template: + src: "templates/bkenv.sh" + dest: "/opt/pulsar/conf/bkenv.sh" - name: Change locale to en_US.utf-8 shell: | echo 'LANG=en_US.utf-8 diff --git a/driver-pulsar/deploy/ssd/extra_vars.yaml b/driver-pulsar/deploy/ssd/extra_vars.yaml index 3d7385a01..f6a9bd0a0 100644 --- a/driver-pulsar/deploy/ssd/extra_vars.yaml +++ b/driver-pulsar/deploy/ssd/extra_vars.yaml @@ -16,11 +16,11 @@ pulsar_version: 2.10.1 node_exporter_version: 1.2.2 prometheus_version: 2.31.1 -pulsar_max_heap_memory: 16g -pulsar_max_direct_memory: 48g +pulsar_max_heap_memory: 32g +pulsar_max_direct_memory: 32g -zookeeper_max_heap_memory: 512m -zookeeper_max_direct_memory: 512m +zookeeper_max_heap_memory: 10g +zookeeper_max_direct_memory: 2g # Set it with true if you want BookKeeper to skip the journal writes. skip_journal: false diff --git a/driver-pulsar/deploy/ssd/templates/bkenv.sh b/driver-pulsar/deploy/ssd/templates/bkenv.sh new file mode 100644 index 000000000..393fb7cb4 --- /dev/null +++ b/driver-pulsar/deploy/ssd/templates/bkenv.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# NOTE: this script is intentionally not executable. It is only meant to be sourced for environment variables. + +# Set JAVA_HOME here to override the environment setting +# JAVA_HOME= + +# default settings for starting bookkeeper + +# Configuration file of settings used in bookie server +BOOKIE_CONF=${BOOKIE_CONF:-"$BK_HOME/conf/bookkeeper.conf"} + +# Log4j configuration file +# BOOKIE_LOG_CONF= + +# Logs location +BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"${PULSAR_LOG_DIR}"} + +# Memory size options +BOOKIE_MEM=${BOOKIE_MEM:-${PULSAR_MEM:-"-Xms{{ max_heap_memory }} -Xmx{{ max_heap_memory }} -XX:MaxDirectMemorySize={{ max_direct_memory }}"}} + +# Garbage collection options +BOOKIE_GC=${BOOKIE_GC:-${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}} + +if [ -z "$JAVA_HOME" ]; then + JAVA_BIN=java +else + JAVA_BIN="$JAVA_HOME/bin/java" +fi +for token in $("$JAVA_BIN" -version 2>&1 | grep 'version "'); do + if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then + if [[ ${BASH_REMATCH[1]} == "1" ]]; then + JAVA_MAJOR_VERSION=${BASH_REMATCH[2]} + else + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + fi + break + elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then + # Process the java versions without dots, such as `17-internal`. + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + break + fi +done + +if [[ -z "$BOOKIE_GC_LOG" ]]; then + # fallback to PULSAR_GC_LOG if it is set + BOOKIE_GC_LOG="$PULSAR_GC_LOG" +fi + +BOOKIE_GC_LOG_DIR=${BOOKIE_GC_LOG_DIR:-"${PULSAR_GC_LOG_DIR:-"${BOOKIE_LOG_DIR}"}"} + +if [[ -z "$BOOKIE_GC_LOG" ]]; then + if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then + BOOKIE_GC_LOG="-Xlog:gc*,safepoint:${BOOKIE_GC_LOG_DIR}/pulsar_bookie_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M" + if [[ $JAVA_MAJOR_VERSION -ge 17 ]]; then + # Use async logging on Java 17+ https://bugs.openjdk.java.net/browse/JDK-8264323 + BOOKIE_GC_LOG="-Xlog:async ${BOOKIE_GC_LOG}" + fi + else + # Java 8 gc log options + BOOKIE_GC_LOG="-Xloggc:${BOOKIE_GC_LOG_DIR}/pulsar_bookie_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M" + fi +fi + +# Extra options to be passed to the jvm +BOOKIE_EXTRA_OPTS="${BOOKIE_EXTRA_OPTS:-"-Dio.netty.leakDetectionLevel=disabled ${PULSAR_EXTRA_OPTS:-"-Dio.netty.recycler.maxCapacityPerThread=4096"}"}" + +# Add extra paths to the bookkeeper classpath +# BOOKIE_EXTRA_CLASSPATH= + +#Folder where the Bookie server PID file should be stored +#BOOKIE_PID_DIR= + +#Wait time before forcefully kill the Bookie server instance, if the stop is not successful +#BOOKIE_STOP_TIMEOUT= + +#Entry formatter class to format entries. +#ENTRY_FORMATTER_CLASS= \ No newline at end of file From cca7698be974215cb03aab7ed64f624bc70e7fba Mon Sep 17 00:00:00 2001 From: Francisco Vila <57452611+franvila@users.noreply.github.com> Date: Thu, 28 Jul 2022 12:06:59 +0200 Subject: [PATCH 027/103] Fixing infinite loop waiting for consumers to be ready (#276) --- .../io/openmessaging/benchmark/WorkloadGenerator.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 2fc7bd1c3..044f30863 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -154,7 +154,9 @@ private void ensureTopicsAreReady() throws IOException { // In this case we just publish 1 message and then wait for consumers to receive the data worker.probeProducers(); - while (true) { + long start = System.currentTimeMillis(); + long end = start + 60 * 1000; + while (System.currentTimeMillis() < end) { CountersStats stats = worker.getCountersStats(); if (stats.messagesReceived < expectedMessages) { @@ -168,7 +170,12 @@ private void ensureTopicsAreReady() throws IOException { } } - log.info("All consumers are ready"); + if (System.currentTimeMillis() >= end) { + log.warn("Timed out waiting for consumers to be ready"); + } + else { + log.info("All consumers are ready"); + } } /** From 31855cdedc1cf28fe20e9550c98285591c6e8ff4 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Sat, 27 Aug 2022 02:41:49 +0800 Subject: [PATCH 028/103] Fix license check and modify configurations for multi topics test (#282) * Fix license check and modify configurations for multi topics test * Update driver-pulsar/pom.xml Co-authored-by: Matteo Merli --- benchmark-framework/pom.xml | 26 +++++----- .../io/openmessaging/benchmark/Benchmark.java | 2 +- .../benchmark/DriverConfiguration.java | 2 +- .../openmessaging/benchmark/ResultsToCsv.java | 2 +- .../openmessaging/benchmark/TestResult.java | 2 +- .../io/openmessaging/benchmark/Workers.java | 2 +- .../io/openmessaging/benchmark/Workload.java | 2 +- .../benchmark/WorkloadGenerator.java | 2 +- .../benchmark/utils/ListPartition.java | 2 +- .../benchmark/utils/PaddingDecimalFormat.java | 3 +- .../benchmark/utils/RandomGenerator.java | 2 +- .../openmessaging/benchmark/utils/Timer.java | 2 +- .../benchmark/utils/UniformRateLimiter.java | 2 +- .../utils/distributor/KeyDistributor.java | 13 +++++ .../utils/distributor/KeyDistributorType.java | 13 +++++ .../utils/distributor/KeyRoundRobin.java | 2 +- .../utils/distributor/NoKeyDistributor.java | 2 +- .../utils/distributor/RandomNano.java | 2 +- .../utils/payload/FilePayloadReader.java | 2 +- .../utils/payload/PayloadException.java | 2 +- .../utils/payload/PayloadReader.java | 2 +- .../benchmark/worker/BenchmarkWorker.java | 2 +- .../worker/DistributedWorkersEnsemble.java | 11 +++-- .../benchmark/worker/LocalWorker.java | 17 ++++--- .../benchmark/worker/Worker.java | 2 +- .../benchmark/worker/WorkerHandler.java | 2 +- .../worker/commands/ConsumerAssignment.java | 2 +- .../worker/commands/CountersStats.java | 2 +- .../worker/commands/CumulativeLatencies.java | 2 +- .../worker/commands/PeriodStats.java | 2 +- .../commands/ProducerWorkAssignment.java | 2 +- .../worker/commands/TopicSubscription.java | 2 +- .../benchmark/worker/commands/TopicsInfo.java | 2 +- docker/pom.xml | 22 +++++---- driver-api/pom.xml | 22 +++++---- .../benchmark/driver/BenchmarkConsumer.java | 2 +- .../benchmark/driver/BenchmarkDriver.java | 2 +- .../benchmark/driver/BenchmarkProducer.java | 2 +- .../benchmark/driver/ConsumerCallback.java | 2 +- driver-artemis/pom.xml | 26 +++++----- .../artemis/ArtemisBenchmarkConsumer.java | 2 +- .../artemis/ArtemisBenchmarkDriver.java | 2 +- .../artemis/ArtemisBenchmarkProducer.java | 2 +- .../driver/artemis/ArtemisConfig.java | 2 +- driver-bookkeeper/pom.xml | 26 +++++----- .../benchmark/driver/bookkeeper/Config.java | 2 +- .../bookkeeper/DlogBenchmarkConsumer.java | 2 +- .../bookkeeper/DlogBenchmarkDriver.java | 2 +- .../bookkeeper/DlogBenchmarkProducer.java | 2 +- .../bookkeeper/stats/CounterAdaptor.java | 2 +- .../driver/bookkeeper/stats/GaugeAdaptor.java | 2 +- .../stats/OpStatsLoggerAdaptor.java | 2 +- .../bookkeeper/stats/StatsLoggerAdaptor.java | 2 +- driver-jms/pom.xml | 22 +++++---- .../driver/jms/JMSBenchmarkConsumer.java | 2 +- .../driver/jms/JMSBenchmarkDriver.java | 3 +- .../driver/jms/JMSBenchmarkProducer.java | 2 +- .../jms/JMSBenchmarkTransactionProducer.java | 2 +- .../driver/jms/config/JMSConfig.java | 2 +- .../deploy/ssd-deployment/deploy.yaml | 2 +- .../deploy/ssd-deployment/terraform.tfvars | 4 +- driver-kafka/kafka-throughput.yaml | 1 + driver-kafka/pom.xml | 24 +++++----- .../benchmark/driver/kafka/Config.java | 2 +- .../driver/kafka/KafkaBenchmarkConsumer.java | 2 +- .../driver/kafka/KafkaBenchmarkDriver.java | 18 +++---- .../driver/kafka/KafkaBenchmarkProducer.java | 13 +++++ driver-kop/pom.xml | 26 +++++----- .../driver/kop/KopBenchmarkDriver.java | 2 +- .../driver/kop/config/ClientType.java | 2 +- .../benchmark/driver/kop/config/Config.java | 2 +- .../driver/kop/config/PulsarConfig.java | 2 +- .../driver/kop/KopBenchmarkDriverTest.java | 2 +- driver-nats-streaming/pom.xml | 22 +++++---- .../NatsStreamingBenchmarkConsumer.java | 2 +- .../NatsStreamingBenchmarkDriver.java | 2 +- .../NatsStreamingBenchmarkProducer.java | 2 +- .../NatsStreamingClientConfig.java | 2 +- driver-nats/pom.xml | 22 +++++---- .../driver/nats/NatsBenchmarkConsumer.java | 2 +- .../driver/nats/NatsBenchmarkDriver.java | 2 +- .../driver/nats/NatsBenchmarkProducer.java | 5 +- .../benchmark/driver/nats/NatsConfig.java | 2 +- driver-nsq/pom.xml | 22 +++++---- .../driver/nsq/NsqBenchmarkConsumer.java | 2 +- .../driver/nsq/NsqBenchmarkDriver.java | 2 +- .../driver/nsq/NsqBenchmarkProducer.java | 2 +- .../benchmark/driver/nsq/NsqConfig.java | 2 +- driver-pravega/deploy/templates/logback.xml | 26 +++++----- driver-pravega/pom.xml | 22 +++++---- .../pravega/PravegaBenchmarkConsumer.java | 2 +- .../pravega/PravegaBenchmarkDriver.java | 2 +- .../pravega/PravegaBenchmarkProducer.java | 2 +- .../PravegaBenchmarkTransactionProducer.java | 2 +- .../pravega/config/PravegaClientConfig.java | 2 +- .../driver/pravega/config/PravegaConfig.java | 2 +- .../pravega/config/PravegaWriterConfig.java | 2 +- driver-pulsar/deploy/ssd/deploy.yaml | 18 +++---- driver-pulsar/deploy/ssd/templates/bkenv.sh | 23 ++++----- .../deploy/ssd/templates/broker.conf | 17 +++---- .../ssd/templates/pulsar-dashboard.service | 2 - .../deploy/ssd/templates/pulsar_env.sh | 47 +++++++++++++++---- driver-pulsar/deploy/ssd/templates/zoo.cfg | 3 +- driver-pulsar/pom.xml | 22 +++++---- .../pulsar/PulsarBenchmarkConsumer.java | 2 +- .../driver/pulsar/PulsarBenchmarkDriver.java | 10 ++-- .../pulsar/PulsarBenchmarkProducer.java | 2 +- .../pulsar/config/PulsarClientConfig.java | 4 +- .../driver/pulsar/config/PulsarConfig.java | 2 +- .../pulsar/config/PulsarConsumerConfig.java | 2 +- .../pulsar/config/PulsarProducerConfig.java | 2 +- driver-rabbitmq/pom.xml | 22 +++++---- .../rabbitmq/RabbitMqBenchmarkConsumer.java | 2 +- .../rabbitmq/RabbitMqBenchmarkDriver.java | 2 +- .../rabbitmq/RabbitMqBenchmarkProducer.java | 2 +- .../driver/rabbitmq/RabbitMqConfig.java | 2 +- driver-redis/pom.xml | 26 +++++----- .../driver/redis/RedisBenchmarkConsumer.java | 2 +- .../driver/redis/RedisBenchmarkDriver.java | 2 +- .../driver/redis/RedisBenchmarkProducer.java | 2 +- .../redis/client/RedisClientConfig.java | 2 +- driver-rocketmq/pom.xml | 26 +++++----- .../rocketmq/RocketMQBenchmarkConsumer.java | 2 +- .../rocketmq/RocketMQBenchmarkDriver.java | 2 +- .../rocketmq/RocketMQBenchmarkProducer.java | 2 +- .../rocketmq/client/RocketMQClientConfig.java | 2 +- package/pom.xml | 22 +++++---- package/src/assemble/bin.xml | 22 +++++---- package/src/assemble/src.xml | 22 +++++---- pom.xml | 26 +++++----- style/copyright/Apache.xml | 22 +++++---- workloads/100-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/100-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/100k-topic-1kb-4p-4c-100k.yaml | 33 +++++++++++++ workloads/100k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/100k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/10k-topic-1kb-4p-4c-100k.yaml | 33 +++++++++++++ workloads/10k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/10k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/1k-topic-1kb-4p-4c-100k.yaml | 33 +++++++++++++ workloads/1k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/1k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/1m-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/1m-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/200k-topic-1kb-4p-4c-100k.yaml | 33 +++++++++++++ workloads/200k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/200k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/300k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/300k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/400k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/400k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/500-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/500-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/500k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/500k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/50k-topic-1kb-4p-4c-100k.yaml | 33 +++++++++++++ workloads/50k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/50k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/5k-topic-1kb-4p-4c-100k.yaml | 33 +++++++++++++ workloads/5k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/5k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/600k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/600k-topic-1kb-4p-4c-600k.yaml | 33 +++++++++++++ workloads/700k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/700k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/800k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/800k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ workloads/900k-topic-1kb-4p-4c-2000k.yaml | 33 +++++++++++++ workloads/900k-topic-1kb-4p-4c-500k.yaml | 33 +++++++++++++ 169 files changed, 1770 insertions(+), 407 deletions(-) create mode 100644 workloads/100-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/100-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/100k-topic-1kb-4p-4c-100k.yaml create mode 100644 workloads/100k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/100k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/10k-topic-1kb-4p-4c-100k.yaml create mode 100644 workloads/10k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/10k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/1k-topic-1kb-4p-4c-100k.yaml create mode 100644 workloads/1k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/1k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/1m-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/1m-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/200k-topic-1kb-4p-4c-100k.yaml create mode 100644 workloads/200k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/200k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/300k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/300k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/400k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/400k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/500-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/500-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/500k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/500k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/50k-topic-1kb-4p-4c-100k.yaml create mode 100644 workloads/50k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/50k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/5k-topic-1kb-4p-4c-100k.yaml create mode 100644 workloads/5k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/5k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/600k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/600k-topic-1kb-4p-4c-600k.yaml create mode 100644 workloads/700k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/700k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/800k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/800k-topic-1kb-4p-4c-500k.yaml create mode 100644 workloads/900k-topic-1kb-4p-4c-2000k.yaml create mode 100644 workloads/900k-topic-1kb-4p-4c-500k.yaml diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index db3dd6ae3..fabb7669b 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -1,16 +1,18 @@ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index 62a4c8865..8d455911f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java index e220cfa2b..4597ab1a3 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java index cb30a5646..42910ea79 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java index 62e4ae396..c13192391 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java index 52da2c530..076b614c7 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java index 759eb09af..a908f577f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 044f30863..97aa25971 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java index 2af6cfca9..745491d01 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java index 31da515ce..7fc674228 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java @@ -1,5 +1,4 @@ - -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java index 2eba74b9e..8848b9a1f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java index d657daf13..e71ffe330 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java index 829e118e9..fdf99c6eb 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java index 1f26d22c1..6ec3f2462 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java @@ -1,3 +1,16 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java index 773e8d4f8..5949a294f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java @@ -1,3 +1,16 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java index 0a3182221..afa8a1ff0 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java index 593f0e5d4..551875e1b 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java index f7265c140..20ecce0d0 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java index 27cd9463f..ea22c8bd9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java index a3887ac31..afb7e4988 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java index 97d82aa80..0318b55e9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java index d6f9d52fa..47e351454 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index 570c6c81e..95ae70945 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -30,6 +30,8 @@ import org.HdrHistogram.Histogram; import org.apache.pulsar.common.util.FutureUtil; import org.asynchttpclient.AsyncHttpClient; +import org.asynchttpclient.DefaultAsyncHttpClientConfig; +import org.asynchttpclient.Dsl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,13 +59,16 @@ public class DistributedWorkersEnsemble implements Worker { private final List producerWorkers; private final List consumerWorkers; - private final AsyncHttpClient httpClient = asyncHttpClient(); + private final AsyncHttpClient httpClient; private int numberOfUsedProducerWorkers; public DistributedWorkersEnsemble(List workers, boolean extraConsumerWorkers) { Preconditions.checkArgument(workers.size() > 1); - + DefaultAsyncHttpClientConfig.Builder clientBuilder = Dsl.config() + .setReadTimeout(600000) + .setRequestTimeout(600000); + httpClient = asyncHttpClient(clientBuilder); this.workers = workers; // For driver-jms extra consumers are required. diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index 3598a5bbd..e34e4c19d 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -87,12 +87,13 @@ public class LocalWorker implements Worker, ConsumerCallback { private final LongAdder totalMessagesSent = new LongAdder(); private final LongAdder totalMessagesReceived = new LongAdder(); - private final Recorder publishLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5); - private final Recorder cumulativePublishLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5); + private final static long highestTrackableValue = TimeUnit.SECONDS.toMicros(60); + private final Recorder publishLatencyRecorder = new Recorder(highestTrackableValue, 5); + private final Recorder cumulativePublishLatencyRecorder = new Recorder(highestTrackableValue, 5); private final OpStatsLogger publishLatencyStats; - private final Recorder publishDelayLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5); - private final Recorder cumulativePublishDelayLatencyRecorder = new Recorder(TimeUnit.SECONDS.toMicros(60), 5); + private final Recorder publishDelayLatencyRecorder = new Recorder(highestTrackableValue, 5); + private final Recorder cumulativePublishDelayLatencyRecorder = new Recorder(highestTrackableValue, 5); private final OpStatsLogger publishDelayLatencyStats; private final Recorder endToEndLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5); @@ -228,12 +229,14 @@ private void submitProducersToExecutor(List producers, KeyDis bytesSent.add(payloadData.length); bytesSentCounter.add(payloadData.length); - long latencyMicros = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sendTime); + final long latencyMicros = Math.min(highestTrackableValue, + TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sendTime)); publishLatencyRecorder.recordValue(latencyMicros); cumulativePublishLatencyRecorder.recordValue(latencyMicros); publishLatencyStats.registerSuccessfulEvent(latencyMicros, TimeUnit.MICROSECONDS); - final long sendDelayMicros = TimeUnit.NANOSECONDS.toMicros(sendTime - intendedSendTime); + final long sendDelayMicros = Math.min(highestTrackableValue, + TimeUnit.NANOSECONDS.toMicros(sendTime - intendedSendTime)); publishDelayLatencyRecorder.recordValue(sendDelayMicros); cumulativePublishDelayLatencyRecorder.recordValue(sendDelayMicros); publishDelayLatencyStats.registerSuccessfulEvent(sendDelayMicros, TimeUnit.MICROSECONDS); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java index 794e5635a..6331c6b79 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java index 951fab4e9..a83bd0d8e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java index 3ae9c97e7..e7a90aa98 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java index 0da132c47..b28474838 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java index 92a12ef7b..bb2484b8a 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java index 1d8c62b73..9f08be837 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java index 53c7e8869..83f431199 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java index f21ce6745..d4eb57e40 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java index b277a8f5f..9b9fc4a0f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/docker/pom.xml b/docker/pom.xml index 331cc2444..bf5188238 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> diff --git a/driver-api/pom.xml b/driver-api/pom.xml index 1ded7a90a..85a7d7fb5 100644 --- a/driver-api/pom.xml +++ b/driver-api/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java index 1788dc6a5..cc2c00385 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java index 0d986f223..99d137312 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java index da9bf06ac..1c2420177 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java index 88817009f..8ae5f7254 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-artemis/pom.xml b/driver-artemis/pom.xml index 7fd18106a..2427c466d 100644 --- a/driver-artemis/pom.xml +++ b/driver-artemis/pom.xml @@ -1,16 +1,18 @@ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java index c714e71fc..09a0ae833 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java index ae49824c1..172b4f4f9 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java index baaa8cf7c..81d8e483a 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java index 418340b66..701bc6638 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/pom.xml b/driver-bookkeeper/pom.xml index fcd920f19..119e1cc1d 100644 --- a/driver-bookkeeper/pom.xml +++ b/driver-bookkeeper/pom.xml @@ -1,16 +1,18 @@ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java index 04b660c50..597080640 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java index 88657e6ee..760c3e848 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java index b1ea6071c..fff244d5e 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java index ed2f667a3..a35f8635a 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java index 61d359969..fdf294a61 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java index 3900ef854..ae3c38422 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java index 4f0c4fbdd..72c580d71 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java index 37337557d..f2627a20a 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/pom.xml b/driver-jms/pom.xml index f79a6462a..8f71eff58 100644 --- a/driver-jms/pom.xml +++ b/driver-jms/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java index cc4730d47..44f6becc5 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java index 5d8683000..12b715cb3 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java @@ -1,5 +1,4 @@ - -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java index 7b5f9ac4b..b24f0313d 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java index fc3ca70a6..60d60c2d7 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java index e550872d5..0a87e401c 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/deploy/ssd-deployment/deploy.yaml b/driver-kafka/deploy/ssd-deployment/deploy.yaml index edc5bf523..d274cb6a6 100644 --- a/driver-kafka/deploy/ssd-deployment/deploy.yaml +++ b/driver-kafka/deploy/ssd-deployment/deploy.yaml @@ -80,7 +80,7 @@ - set_fact: zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" boostrapServers: "{{ groups['kafka'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:9092') | join(',') }}" - kafkaVersion: "3.0.0" + kafkaVersion: "3.2.1" - debug: msg: "zookeeper servers: {{ zookeeperServers }}\nboostrap servers: {{ boostrapServers }}" - name: Download Kafka package diff --git a/driver-kafka/deploy/ssd-deployment/terraform.tfvars b/driver-kafka/deploy/ssd-deployment/terraform.tfvars index ac01d4acb..09e97c92f 100644 --- a/driver-kafka/deploy/ssd-deployment/terraform.tfvars +++ b/driver-kafka/deploy/ssd-deployment/terraform.tfvars @@ -1,11 +1,11 @@ public_key_path = "~/.ssh/kafka_aws.pub" region = "us-west-2" az = "us-west-2a" -ami = "ami-9fa343e7" // RHEL-7.4 +ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { "kafka" = "i3en.6xlarge" - "zookeeper" = "t2.small" + "zookeeper" = "t2.xlarge" "client" = "m5n.8xlarge" } diff --git a/driver-kafka/kafka-throughput.yaml b/driver-kafka/kafka-throughput.yaml index 48901e246..445202bd2 100644 --- a/driver-kafka/kafka-throughput.yaml +++ b/driver-kafka/kafka-throughput.yaml @@ -25,6 +25,7 @@ topicConfig: | commonConfig: | bootstrap.servers=localhost:9092 + default.api.timeout.ms=600000 producerConfig: | acks=all diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index 902e68ddf..e87b7d812 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 @@ -32,7 +34,7 @@ org.apache.kafka kafka-clients - 3.0.0 + 3.2.1 diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java index 710580344..7f28aaf57 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java index 171c852e5..9b8d1f7d5 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java index 1b1f207df..a87ffe8de 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -110,15 +110,17 @@ public String getTopicNamePrefix() { @SuppressWarnings({ "rawtypes", "unchecked" }) @Override public CompletableFuture createTopic(String topic, int partitions) { - return CompletableFuture.runAsync(() -> { - try { - NewTopic newTopic = new NewTopic(topic, partitions, config.replicationFactor); - newTopic.configs(new HashMap<>((Map) topicProperties)); - admin.createTopics(Arrays.asList(newTopic)).all().get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); + NewTopic newTopic = new NewTopic(topic, partitions, config.replicationFactor); + newTopic.configs(new HashMap<>((Map) topicProperties)); + CompletableFuture promise = new CompletableFuture<>(); + admin.createTopics(Collections.singletonList(newTopic)).all().whenComplete((__, e) -> { + if (e != null) { + promise.completeExceptionally(e); + } else { + promise.complete(null); } }); + return promise; } @Override diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java index 7faba69c9..f242bc2af 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java @@ -1,3 +1,16 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/driver-kop/pom.xml b/driver-kop/pom.xml index cf78b95a5..6090951f8 100644 --- a/driver-kop/pom.xml +++ b/driver-kop/pom.xml @@ -1,16 +1,18 @@ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java index d855d7c84..b87163455 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java index 1e25a856e..f2e88a386 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java index daff56a9c..1a8daeb8f 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java index d9c36bc09..595a504d9 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java index b1fbdffb2..403515b7f 100644 --- a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java +++ b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats-streaming/pom.xml b/driver-nats-streaming/pom.xml index 28f1bb281..40a876e2e 100644 --- a/driver-nats-streaming/pom.xml +++ b/driver-nats-streaming/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java index 8c5fd250b..062dce260 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index dadc4d278..5f1693057 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java index bfcba157d..1492404a5 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,16 +13,13 @@ */ package io.openmessaging.benchmark.driver.nats; -import io.nats.client.ConnectionListener; import io.openmessaging.benchmark.driver.BenchmarkProducer; -import java.time.Duration; import java.util.Optional; import java.util.concurrent.CompletableFuture; import io.nats.client.Connection; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; -import javax.sql.ConnectionEventListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java index 6fb51d4a1..ec2e40626 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nsq/pom.xml b/driver-nsq/pom.xml index 5949884a5..5f16cf095 100644 --- a/driver-nsq/pom.xml +++ b/driver-nsq/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java index a301da2ca..4012a41e0 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java index 4c1581c89..e0e0d0a88 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java index 1ae87645e..701d42950 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java index 5658105e2..c5e8189a7 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/deploy/templates/logback.xml b/driver-pravega/deploy/templates/logback.xml index 33bec7685..ad30f68d0 100644 --- a/driver-pravega/deploy/templates/logback.xml +++ b/driver-pravega/deploy/templates/logback.xml @@ -1,17 +1,19 @@ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> diff --git a/driver-pravega/pom.xml b/driver-pravega/pom.xml index 6989533a8..dea4a10ce 100644 --- a/driver-pravega/pom.xml +++ b/driver-pravega/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java index d2b12b252..20928b96c 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java index 2f8eefa53..15cfd0e8f 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java index fd9546fd3..e59849408 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java index 8071caec3..883cf5100 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java index f1dd86b60..be69af873 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java index 0853bb4b3..30b7ac5e3 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java index d470e8d31..1466c7068 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index 2e53000f8..b42204275 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -218,21 +218,21 @@ state: restarted daemon_reload: yes name: "zookeeper" - - command: > - bin/pulsar initialize-cluster-metadata --cluster local - --zookeeper localhost:2181 - --configuration-store localhost:2181 - --web-service-url {{ httpUrl }} - --broker-service-url {{ serviceUrl }} - args: - chdir: /opt/pulsar - when: groups['zookeeper'][0] == inventory_hostname - name: BookKeeper setup hosts: pulsar connection: ssh become: true tasks: + - command: > + bin/pulsar initialize-cluster-metadata --cluster local + --zookeeper {{ hostvars[groups['zookeeper'][0]].private_ip }}:2181 + --configuration-store {{ hostvars[groups['zookeeper'][0]].private_ip }}:2181 + --web-service-url {{ httpUrl }} + --broker-service-url {{ serviceUrl }} + args: + chdir: /opt/pulsar + when: groups['pulsar'][0] == inventory_hostname - set_fact: skip_journal: "{{ skip_journal | default(false) }}" - name: Apply BookKeeper configuration file [Bypass Journal] diff --git a/driver-pulsar/deploy/ssd/templates/bkenv.sh b/driver-pulsar/deploy/ssd/templates/bkenv.sh index 393fb7cb4..ab530c297 100644 --- a/driver-pulsar/deploy/ssd/templates/bkenv.sh +++ b/driver-pulsar/deploy/ssd/templates/bkenv.sh @@ -1,21 +1,16 @@ #!/bin/sh # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # NOTE: this script is intentionally not executable. It is only meant to be sourced for environment variables. diff --git a/driver-pulsar/deploy/ssd/templates/broker.conf b/driver-pulsar/deploy/ssd/templates/broker.conf index 6397826df..bea319ea0 100644 --- a/driver-pulsar/deploy/ssd/templates/broker.conf +++ b/driver-pulsar/deploy/ssd/templates/broker.conf @@ -18,28 +18,23 @@ # Zookeeper quorum connection string zookeeperServers={{ zookeeperServers }} - # Global Zookeeper quorum connection string globalZookeeperServers={{ zookeeperServers }} - # Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used. advertisedAddress={{ hostvars[inventory_hostname].private_ip }} - # Name of the cluster to which this broker belongs to clusterName=local - loadBalancerAutoUnloadSplitBundlesEnabled=false - defaultNumberOfNamespaceBundles=64 - managedLedgerNewEntriesCheckDelayInMillis=0 - bookkeeperNumberOfChannelsPerBookie=64 - managedLedgerCacheEvictionFrequency=0.2 - # Increasing from default to avoid throttling lookups maxConcurrentLookupRequest=500000 - loadBalancerOverrideBrokerNicSpeedGbps=25 - +forceDeleteTenantAllowed=true +forceDeleteNamespaceAllowed=true +exposeManagedLedgerMetricsInPrometheus=false +exposeTopicLevelMetricsInPrometheus=false +systemTopicEnabled=false +topicLevelPoliciesEnabled=false diff --git a/driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service b/driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service index 24e69b7f7..a3ee6a0ef 100644 --- a/driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service +++ b/driver-pulsar/deploy/ssd/templates/pulsar-dashboard.service @@ -1,7 +1,5 @@ [Unit] Description=Pulsar Dashboard -After=docker.service -Requires=docker.service After=prometheus.service Requires=prometheus.service diff --git a/driver-pulsar/deploy/ssd/templates/pulsar_env.sh b/driver-pulsar/deploy/ssd/templates/pulsar_env.sh index b33df2ee6..f13d0332c 100644 --- a/driver-pulsar/deploy/ssd/templates/pulsar_env.sh +++ b/driver-pulsar/deploy/ssd/templates/pulsar_env.sh @@ -37,17 +37,48 @@ # PULSAR_GLOBAL_ZK_CONF= # Extra options to be passed to the jvm -PULSAR_MEM=" -Xms{{ max_heap_memory }} -Xmx{{ max_heap_memory }} -XX:MaxDirectMemorySize={{ max_direct_memory }}" +PULSAR_MEM=${PULSAR_MEM:-" -Xms{{ max_heap_memory }} -Xmx{{ max_heap_memory }} -XX:MaxDirectMemorySize={{ max_direct_memory }}"} # Garbage collection options -PULSAR_GC=" -XX:+UseZGC" -PULSAR_GC="${PULSAR_GC} -XX:+PerfDisableSharedMem" +PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"} -# Extra options to be passed to the jvm -PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" +if [ -z "$JAVA_HOME" ]; then + JAVA_BIN=java +else + JAVA_BIN="$JAVA_HOME/bin/java" +fi +for token in $("$JAVA_BIN" -version 2>&1 | grep 'version "'); do + if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then + if [[ ${BASH_REMATCH[1]} == "1" ]]; then + JAVA_MAJOR_VERSION=${BASH_REMATCH[2]} + else + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + fi + break + elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then + # Process the java versions without dots, such as `17-internal`. + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + break + fi +done + +PULSAR_GC_LOG_DIR=${PULSAR_GC_LOG_DIR:-"${PULSAR_LOG_DIR}"} -# Extra options to be passed to the BookKeeper JVM -BOOKIE_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" +if [[ -z "$PULSAR_GC_LOG" ]]; then + if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then + PULSAR_GC_LOG="-Xlog:gc*,safepoint:${PULSAR_GC_LOG_DIR}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M" + if [[ $JAVA_MAJOR_VERSION -ge 17 ]]; then + # Use async logging on Java 17+ https://bugs.openjdk.java.net/browse/JDK-8264323 + PULSAR_GC_LOG="-Xlog:async ${PULSAR_GC_LOG}" + fi + else + # Java 8 gc log options + PULSAR_GC_LOG="-Xloggc:${PULSAR_GC_LOG_DIR}/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M" + fi +fi + +# Extra options to be passed to the jvm +PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS:-" -Dpulsar.allocator.exit_on_oom=true -Dio.netty.recycler.maxCapacityPerThread=4096"}" # Add extra paths to the bookkeeper classpath # PULSAR_EXTRA_CLASSPATH= @@ -55,5 +86,5 @@ BOOKIE_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.le #Folder where the Bookie server PID file should be stored #PULSAR_PID_DIR= -#Wait time before forcefully kill the pulser server instance, if the stop is not successful +#Wait time before forcefully kill the pulsar server instance, if the stop is not successful #PULSAR_STOP_TIMEOUT= diff --git a/driver-pulsar/deploy/ssd/templates/zoo.cfg b/driver-pulsar/deploy/ssd/templates/zoo.cfg index dbd337067..195e73a08 100644 --- a/driver-pulsar/deploy/ssd/templates/zoo.cfg +++ b/driver-pulsar/deploy/ssd/templates/zoo.cfg @@ -38,7 +38,8 @@ autopurge.snapRetainCount=3 # Purge task interval in hours # Set to "0" to disable auto purge feature autopurge.purgeInterval=1 - +metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider +metricsProvider.httpPort=8000 {% for zk in groups['zookeeper'] %} server.{{ hostvars[zk].zid }}={{ hostvars[zk].private_ip }}:2888:3888 {% endfor %} diff --git a/driver-pulsar/pom.xml b/driver-pulsar/pom.xml index 5606703d8..839ebf811 100644 --- a/driver-pulsar/pom.xml +++ b/driver-pulsar/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java index f64db5f62..1d73ea9a6 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java index 0d085dba0..ba4a06bf3 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java @@ -1,5 +1,4 @@ - -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -66,8 +65,6 @@ public class PulsarBenchmarkDriver implements BenchmarkDriver { private PulsarConfig config; - - private String namespace; private ProducerBuilder producerBuilder; @@ -81,9 +78,10 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I .connectionsPerBroker(config.client.connectionsPerBroker) .statsInterval(0, TimeUnit.SECONDS) .serviceUrl(config.client.serviceUrl) - .maxConcurrentLookupRequests(50000) - .maxLookupRequests(100000) + .maxConcurrentLookupRequests(config.client.maxConcurrentLookupRequests) + .maxLookupRequests(Integer.MAX_VALUE) .memoryLimit(config.client.clientMemoryLimitMB, SizeUnit.MEGA_BYTES) + .operationTimeout(10, TimeUnit.MINUTES) .listenerThreads(Runtime.getRuntime().availableProcessors()); if (config.client.serviceUrl.startsWith("pulsar+ssl")) { diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java index cad7a4a12..cda35459f 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java index a143e75a0..cbdebd39a 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -26,6 +26,8 @@ public class PulsarClientConfig { public int connectionsPerBroker = 8; + public int maxConcurrentLookupRequests = 1000; + public String namespacePrefix; public String clusterName; diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java index 8bcb24622..ec3512c63 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java index 981271eec..8d34d8819 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java index 0cfa555f4..1528e4d74 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rabbitmq/pom.xml b/driver-rabbitmq/pom.xml index e30c157d5..1828f94da 100644 --- a/driver-rabbitmq/pom.xml +++ b/driver-rabbitmq/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java index 975121e18..a7b30925a 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index 328bd1d0f..370f7ae76 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java index f795158ee..7aeaaeb3d 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java index 6518e44fe..8fe2794cb 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-redis/pom.xml b/driver-redis/pom.xml index 105a07c44..2b3e68361 100644 --- a/driver-redis/pom.xml +++ b/driver-redis/pom.xml @@ -1,16 +1,18 @@ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java index c9d8b60a2..73d41e4cf 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java index 9e5b28c87..0e8b17215 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java index d62c4c246..71ca82802 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java index 899f752ef..6bb220607 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rocketmq/pom.xml b/driver-rocketmq/pom.xml index dc388dd56..bc812c6bf 100644 --- a/driver-rocketmq/pom.xml +++ b/driver-rocketmq/pom.xml @@ -1,16 +1,18 @@ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java index a79b9ede1..9e483ae71 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java index 8ac7c1f72..6efefa2ec 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java index 838808729..811e05d66 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java index 840090452..a597c4057 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java @@ -1,4 +1,4 @@ -/* +/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/package/pom.xml b/package/pom.xml index aed395e9f..07235dfba 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/package/src/assemble/bin.xml b/package/src/assemble/bin.xml index 19cece2ad..1ae16f685 100644 --- a/package/src/assemble/bin.xml +++ b/package/src/assemble/bin.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> 4.0.0 diff --git a/style/copyright/Apache.xml b/style/copyright/Apache.xml index 8301fbd80..1dc9ce264 100644 --- a/style/copyright/Apache.xml +++ b/style/copyright/Apache.xml @@ -1,16 +1,18 @@ + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> diff --git a/workloads/100-topic-1kb-4p-4c-2000k.yaml b/workloads/100-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..f70d0da2b --- /dev/null +++ b/workloads/100-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 100 topic + +topics: 100 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/100-topic-1kb-4p-4c-500k.yaml b/workloads/100-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..4b577432b --- /dev/null +++ b/workloads/100-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 100 topic + +topics: 100 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/100k-topic-1kb-4p-4c-100k.yaml b/workloads/100k-topic-1kb-4p-4c-100k.yaml new file mode 100644 index 000000000..20c6b04f2 --- /dev/null +++ b/workloads/100k-topic-1kb-4p-4c-100k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 100k rate 4 producers and 4 consumers on 100k topic + +topics: 100000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 100000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/100k-topic-1kb-4p-4c-2000k.yaml b/workloads/100k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..6b5134436 --- /dev/null +++ b/workloads/100k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 100k topic + +topics: 100000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/100k-topic-1kb-4p-4c-500k.yaml b/workloads/100k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..4a5db4461 --- /dev/null +++ b/workloads/100k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 100k topic + +topics: 100000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/10k-topic-1kb-4p-4c-100k.yaml b/workloads/10k-topic-1kb-4p-4c-100k.yaml new file mode 100644 index 000000000..365790b42 --- /dev/null +++ b/workloads/10k-topic-1kb-4p-4c-100k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 100k rate 4 producers and 4 consumers on 10k topic + +topics: 10000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 100000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/10k-topic-1kb-4p-4c-2000k.yaml b/workloads/10k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..c007225fa --- /dev/null +++ b/workloads/10k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 10k topic + +topics: 10000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/10k-topic-1kb-4p-4c-500k.yaml b/workloads/10k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..b48f9e7e6 --- /dev/null +++ b/workloads/10k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 10k topic + +topics: 10000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/1k-topic-1kb-4p-4c-100k.yaml b/workloads/1k-topic-1kb-4p-4c-100k.yaml new file mode 100644 index 000000000..ccf38b9f0 --- /dev/null +++ b/workloads/1k-topic-1kb-4p-4c-100k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 100k rate 4 producers and 4 consumers on 1k topic + +topics: 1000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 100000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/1k-topic-1kb-4p-4c-2000k.yaml b/workloads/1k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..b060d1295 --- /dev/null +++ b/workloads/1k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 1k topic + +topics: 1000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/1k-topic-1kb-4p-4c-500k.yaml b/workloads/1k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..1f389de50 --- /dev/null +++ b/workloads/1k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 1k topic + +topics: 1000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/1m-topic-1kb-4p-4c-2000k.yaml b/workloads/1m-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..051ae6e01 --- /dev/null +++ b/workloads/1m-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 1m topic + +topics: 1000000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/1m-topic-1kb-4p-4c-500k.yaml b/workloads/1m-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..f1a107564 --- /dev/null +++ b/workloads/1m-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 1m topic + +topics: 1000000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/200k-topic-1kb-4p-4c-100k.yaml b/workloads/200k-topic-1kb-4p-4c-100k.yaml new file mode 100644 index 000000000..7fd75dc8d --- /dev/null +++ b/workloads/200k-topic-1kb-4p-4c-100k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 100k rate 4 producers and 4 consumers on 200k topic + +topics: 200000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 100000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/200k-topic-1kb-4p-4c-2000k.yaml b/workloads/200k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..7d737bce2 --- /dev/null +++ b/workloads/200k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 200k topic + +topics: 200000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/200k-topic-1kb-4p-4c-500k.yaml b/workloads/200k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..83ca06336 --- /dev/null +++ b/workloads/200k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 200k topic + +topics: 200000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/300k-topic-1kb-4p-4c-2000k.yaml b/workloads/300k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..cfcfb12fc --- /dev/null +++ b/workloads/300k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 300k topic + +topics: 300000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/300k-topic-1kb-4p-4c-500k.yaml b/workloads/300k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..86482683f --- /dev/null +++ b/workloads/300k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 300k topic + +topics: 300000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/400k-topic-1kb-4p-4c-2000k.yaml b/workloads/400k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..c7286c416 --- /dev/null +++ b/workloads/400k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 400k topic + +topics: 400000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/400k-topic-1kb-4p-4c-500k.yaml b/workloads/400k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..acea013b2 --- /dev/null +++ b/workloads/400k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 400k topic + +topics: 400000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/500-topic-1kb-4p-4c-2000k.yaml b/workloads/500-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..c8bb3795f --- /dev/null +++ b/workloads/500-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 500 topic + +topics: 500 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/500-topic-1kb-4p-4c-500k.yaml b/workloads/500-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..0dc59c443 --- /dev/null +++ b/workloads/500-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 500 topic + +topics: 500 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/500k-topic-1kb-4p-4c-2000k.yaml b/workloads/500k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..42030106f --- /dev/null +++ b/workloads/500k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 500k topic + +topics: 500000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/500k-topic-1kb-4p-4c-500k.yaml b/workloads/500k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..addfcc98d --- /dev/null +++ b/workloads/500k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 500k topic + +topics: 500000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/50k-topic-1kb-4p-4c-100k.yaml b/workloads/50k-topic-1kb-4p-4c-100k.yaml new file mode 100644 index 000000000..e38e77acb --- /dev/null +++ b/workloads/50k-topic-1kb-4p-4c-100k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 100k rate 4 producers and 4 consumers on 50k topic + +topics: 50000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 100000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/50k-topic-1kb-4p-4c-2000k.yaml b/workloads/50k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..5c066e3a3 --- /dev/null +++ b/workloads/50k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 50k topic + +topics: 50000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/50k-topic-1kb-4p-4c-500k.yaml b/workloads/50k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..2345d64ee --- /dev/null +++ b/workloads/50k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 50k topic + +topics: 50000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/5k-topic-1kb-4p-4c-100k.yaml b/workloads/5k-topic-1kb-4p-4c-100k.yaml new file mode 100644 index 000000000..b092ccedc --- /dev/null +++ b/workloads/5k-topic-1kb-4p-4c-100k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 100k rate 4 producers and 4 consumers on 5k topic + +topics: 5000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 100000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/5k-topic-1kb-4p-4c-2000k.yaml b/workloads/5k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..7c1672926 --- /dev/null +++ b/workloads/5k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 5k topic + +topics: 5000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/5k-topic-1kb-4p-4c-500k.yaml b/workloads/5k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..30e1c80d3 --- /dev/null +++ b/workloads/5k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 5k topic + +topics: 5000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/600k-topic-1kb-4p-4c-2000k.yaml b/workloads/600k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..495d39936 --- /dev/null +++ b/workloads/600k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 600k topic + +topics: 600000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/600k-topic-1kb-4p-4c-600k.yaml b/workloads/600k-topic-1kb-4p-4c-600k.yaml new file mode 100644 index 000000000..9e7231d3a --- /dev/null +++ b/workloads/600k-topic-1kb-4p-4c-600k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 600k topic + +topics: 600000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/700k-topic-1kb-4p-4c-2000k.yaml b/workloads/700k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..e404145f4 --- /dev/null +++ b/workloads/700k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 700k topic + +topics: 700000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/700k-topic-1kb-4p-4c-500k.yaml b/workloads/700k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..d3a44cef9 --- /dev/null +++ b/workloads/700k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 700k topic + +topics: 700000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/800k-topic-1kb-4p-4c-2000k.yaml b/workloads/800k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..a98dcbec0 --- /dev/null +++ b/workloads/800k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 800k topic + +topics: 800000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/800k-topic-1kb-4p-4c-500k.yaml b/workloads/800k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..dfae1b720 --- /dev/null +++ b/workloads/800k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 800k topic + +topics: 800000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/900k-topic-1kb-4p-4c-2000k.yaml b/workloads/900k-topic-1kb-4p-4c-2000k.yaml new file mode 100644 index 000000000..c5674a385 --- /dev/null +++ b/workloads/900k-topic-1kb-4p-4c-2000k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 2000k rate 4 producers and 4 consumers on 900k topic + +topics: 900000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 2000000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file diff --git a/workloads/900k-topic-1kb-4p-4c-500k.yaml b/workloads/900k-topic-1kb-4p-4c-500k.yaml new file mode 100644 index 000000000..4c5ec487e --- /dev/null +++ b/workloads/900k-topic-1kb-4p-4c-500k.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: 500k rate 4 producers and 4 consumers on 900k topic + +topics: 900000 +partitionsPerTopic: 1 + +messageSize: 1024 +useRandomizedPayloads: true +randomBytesRatio: 0.5 +randomizedPayloadPoolSize: 1000 + +subscriptionsPerTopic: 1 +consumerPerSubscription: 4 +producersPerTopic: 4 + +# Discover max-sustainable rate +producerRate: 500000 + +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 \ No newline at end of file From df0d9d3d8cff80eeb628e197023f4bd6454bd5d6 Mon Sep 17 00:00:00 2001 From: Francisco Vila <57452611+franvila@users.noreply.github.com> Date: Fri, 26 Aug 2022 20:42:19 +0200 Subject: [PATCH 029/103] [issue #275] fix waiting for consumers to be ready (#277) * Fixing infinite loop waiting for consumers to be ready * Throw an exception when the timeout is raised when waiting for consumers to be ready --- .../main/java/io/openmessaging/benchmark/WorkloadGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 97aa25971..609737152 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -171,7 +171,7 @@ private void ensureTopicsAreReady() throws IOException { } if (System.currentTimeMillis() >= end) { - log.warn("Timed out waiting for consumers to be ready"); + throw new RuntimeException("Timed out waiting for consumers to be ready"); } else { log.info("All consumers are ready"); From 8294b0a38c3aae0c6e93dcfaf0cec8f378077f09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Aug 2022 11:42:51 -0700 Subject: [PATCH 030/103] Bump jetty-server in /benchmark-framework (#274) Bumps [jetty-server](https://github.com/eclipse/jetty.project) from 9.4.42.v20210604 to 10.0.10. - [Release notes](https://github.com/eclipse/jetty.project/releases) - [Commits](https://github.com/eclipse/jetty.project/compare/jetty-9.4.42.v20210604...jetty-10.0.10) --- updated-dependencies: - dependency-name: org.eclipse.jetty:jetty-server dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- benchmark-framework/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index fabb7669b..f8e4a1298 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -194,7 +194,7 @@ org.eclipse.jetty jetty-server - 9.4.42.v20210604 + 10.0.10 From 94178fac8e7c1c0f0867bbd68cf8aa9b04e50fca Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Tue, 6 Sep 2022 04:56:57 +0800 Subject: [PATCH 031/103] Change zookeeper node type to i3en.2xlarge for multi-topics test (#287) --- .../deploy/ssd-deployment/deploy.yaml | 28 ++++++++++++++++--- .../templates/zookeeper.properties | 3 +- .../templates/zookeeper.service | 1 + .../deploy/ssd-deployment/terraform.tfvars | 2 +- driver-pulsar/deploy/ssd/deploy.yaml | 27 ++++++++++++++++-- driver-pulsar/deploy/ssd/templates/zoo.cfg | 3 +- driver-pulsar/deploy/ssd/terraform.tfvars | 2 +- 7 files changed, 56 insertions(+), 10 deletions(-) diff --git a/driver-kafka/deploy/ssd-deployment/deploy.yaml b/driver-kafka/deploy/ssd-deployment/deploy.yaml index d274cb6a6..80a06392e 100644 --- a/driver-kafka/deploy/ssd-deployment/deploy.yaml +++ b/driver-kafka/deploy/ssd-deployment/deploy.yaml @@ -62,6 +62,29 @@ - name: Reboot the machine with all defaults reboot: +- name: Format and mount disks for Zookeeper hosts + hosts: zookeeper + connection: ssh + become: true + tasks: + - name: Format disks + filesystem: + fstype: xfs + dev: '{{ item }}' + with_items: + - '/dev/nvme1n1' + - '/dev/nvme2n1' + - name: Mount disks + mount: + path: "{{ item.path }}" + src: "{{ item.src }}" + fstype: xfs + opts: defaults,noatime,nodiscard + state: mounted + with_items: + - { path: "/mnt/zookeeper/logs", src: "/dev/nvme1n1" } + - { path: "/mnt/zookeeper/data", src: "/dev/nvme2n1" } + - name: Kafka setup hosts: all connection: ssh @@ -103,15 +126,12 @@ tasks: - set_fact: zid: "{{ groups['zookeeper'].index(inventory_hostname) }}" - - file: - path: "/opt/kafka/data/zookeeper" - state: directory - template: src: "templates/zookeeper.properties" dest: "/opt/kafka/config/zookeeper.properties" - template: src: templates/myid - dest: "/opt/kafka/data/zookeeper/myid" + dest: "/mnt/zookeeper/data/myid" - template: src: "templates/zookeeper.service" dest: "/etc/systemd/system/zookeeper.service" diff --git a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties index dbd337067..4fb919bd5 100644 --- a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties +++ b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.properties @@ -21,7 +21,8 @@ initLimit=10 # sending a request and getting an acknowledgement syncLimit=5 # the directory where the snapshot is stored. -dataDir=data/zookeeper +dataLogDir=/mnt/zookeeper/logs +dataDir=/mnt/zookeeper/data # the port at which the clients will connect clientPort=2181 # the maximum number of client connections. diff --git a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.service b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.service index 3a2706efa..cb0d13d7b 100644 --- a/driver-kafka/deploy/ssd-deployment/templates/zookeeper.service +++ b/driver-kafka/deploy/ssd-deployment/templates/zookeeper.service @@ -3,6 +3,7 @@ Description=ZooKeeper After=network.target [Service] +Environment='KAFKA_HEAP_OPTS=-Xms32g -Xmx32g' ExecStart=/opt/kafka/bin/zookeeper-server-start.sh config/zookeeper.properties WorkingDirectory=/opt/kafka RestartSec=1s diff --git a/driver-kafka/deploy/ssd-deployment/terraform.tfvars b/driver-kafka/deploy/ssd-deployment/terraform.tfvars index 09e97c92f..f6b6d8995 100644 --- a/driver-kafka/deploy/ssd-deployment/terraform.tfvars +++ b/driver-kafka/deploy/ssd-deployment/terraform.tfvars @@ -5,7 +5,7 @@ ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { "kafka" = "i3en.6xlarge" - "zookeeper" = "t2.xlarge" + "zookeeper" = "i3en.2xlarge" "client" = "m5n.8xlarge" } diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index b42204275..f1242da04 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -43,6 +43,29 @@ remote: yes when: prometheus_binary is not defined +- name: Format and mount disks for Zookeeper hosts + hosts: zookeeper + connection: ssh + become: true + tasks: + - name: Format disks + filesystem: + fstype: xfs + dev: '{{ item }}' + with_items: + - "{{ disk_dev[0] }}" + - "{{ disk_dev[1] }}" + - name: Mount disks + mount: + path: "{{ item.path }}" + src: "{{ item.src }}" + fstype: xfs + opts: defaults,noatime,nodiscard + state: mounted + with_items: + - { path: "/mnt/zookeeper/logs", src: "{{ disk_dev[0] }}" } + - { path: "/mnt/zookeeper/data", src: "{{ disk_dev[1] }}" } + - name: Format and mount disks for Pulsar/BookKeeper hosts hosts: pulsar connection: ssh @@ -195,7 +218,7 @@ tasks: - set_fact: zid: "{{ groups['zookeeper'].index(inventory_hostname) }}" - max_heap_memory: "{{ zookeeper_max_heap_memory | default('10G') }}" + max_heap_memory: "{{ zookeeper_max_heap_memory | default('32G') }}" max_direct_memory: "{{ zookeeper_max_direct_memory | default('2G') }}" - file: path: "/opt/pulsar/{{ item }}" @@ -210,7 +233,7 @@ dest: "/opt/pulsar/conf/zookeeper.conf" - template: src: templates/myid - dest: "/opt/pulsar/data/zookeeper/myid" + dest: "/mnt/zookeeper/data/myid" - template: src: "templates/zookeeper.service" dest: "/etc/systemd/system/zookeeper.service" diff --git a/driver-pulsar/deploy/ssd/templates/zoo.cfg b/driver-pulsar/deploy/ssd/templates/zoo.cfg index 195e73a08..56097ca6a 100644 --- a/driver-pulsar/deploy/ssd/templates/zoo.cfg +++ b/driver-pulsar/deploy/ssd/templates/zoo.cfg @@ -21,7 +21,8 @@ initLimit=10 # sending a request and getting an acknowledgement syncLimit=5 # the directory where the snapshot is stored. -dataDir=data/zookeeper +dataLogDir=/mnt/zookeeper/logs +dataDir=/mnt/zookeeper/data # the port at which the clients will connect clientPort=2181 # the maximum number of client connections. diff --git a/driver-pulsar/deploy/ssd/terraform.tfvars b/driver-pulsar/deploy/ssd/terraform.tfvars index 84503c1ec..bc4a9fe2e 100644 --- a/driver-pulsar/deploy/ssd/terraform.tfvars +++ b/driver-pulsar/deploy/ssd/terraform.tfvars @@ -5,7 +5,7 @@ ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { "pulsar" = "i3en.6xlarge" - "zookeeper" = "t2.xlarge" + "zookeeper" = "i3en.2xlarge" "client" = "m5n.8xlarge" "prometheus" = "t2.large" } From 7c55e60aaecfa001ba69c7d31cfb647e53a61e94 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 8 Sep 2022 16:19:10 +0100 Subject: [PATCH 032/103] Revert "Bump jetty-server in /benchmark-framework (#274)" (#289) This reverts commit 8294b0a38c3aae0c6e93dcfaf0cec8f378077f09. --- benchmark-framework/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index f8e4a1298..fabb7669b 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -194,7 +194,7 @@ org.eclipse.jetty jetty-server - 10.0.10 + 9.4.42.v20210604 From 53108b5821757dde1e90a36b132fc3e1b078194f Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Mon, 12 Sep 2022 23:26:54 +0100 Subject: [PATCH 033/103] Create topics in batches to minimise requests and retry failures (#288) --- .../benchmark/worker/LocalWorker.java | 21 +-- driver-api/pom.xml | 8 +- .../benchmark/driver/BenchmarkDriver.java | 19 +++ driver-kafka/pom.xml | 30 +++++ .../driver/kafka/KafkaBenchmarkDriver.java | 22 ++-- .../driver/kafka/KafkaTopicCreator.java | 105 +++++++++++++++ .../driver/kafka/KafkaTopicCreatorTest.java | 124 ++++++++++++++++++ .../test/resources/simplelogger.properties | 18 +++ pom.xml | 7 +- 9 files changed, 329 insertions(+), 25 deletions(-) create mode 100644 driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java create mode 100644 driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java create mode 100644 driver-kafka/src/test/resources/simplelogger.properties diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index e34e4c19d..3e1c37c1a 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -16,6 +16,7 @@ import static io.openmessaging.benchmark.utils.UniformRateLimiter.*; import static java.util.stream.Collectors.toList; +import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -28,6 +29,7 @@ import java.util.concurrent.atomic.LongAdder; import io.openmessaging.benchmark.utils.UniformRateLimiter; +import java.util.stream.IntStream; import org.HdrHistogram.Recorder; import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.NullStatsLogger; @@ -142,25 +144,24 @@ public void initializeDriver(File driverConfigFile) throws IOException { @Override public List createTopics(TopicsInfo topicsInfo) { - List> futures = new ArrayList<>(); - Timer timer = new Timer(); + List topicInfos = IntStream.range(0, topicsInfo.numberOfTopics) + .mapToObj(i -> new TopicInfo(generateTopicName(i), topicsInfo.numberOfPartitionsPerTopic)) + .collect(toList()); - List topics = new ArrayList<>(); - for (int i = 0; i < topicsInfo.numberOfTopics; i++) { - String topicPrefix = benchmarkDriver.getTopicNamePrefix(); - String topic = String.format("%s-%s-%04d", topicPrefix, RandomGenerator.getRandomString(), i); - topics.add(topic); - futures.add(benchmarkDriver.createTopic(topic, topicsInfo.numberOfPartitionsPerTopic)); - } + benchmarkDriver.createTopics(topicInfos).join(); - futures.forEach(CompletableFuture::join); + List topics = topicInfos.stream().map(TopicInfo::getTopic).collect(toList()); log.info("Created {} topics in {} ms", topics.size(), timer.elapsedMillis()); return topics; } + private String generateTopicName(int i) { + return String.format("%s-%s-%07d", benchmarkDriver.getTopicNamePrefix(), RandomGenerator.getRandomString(), i); + } + @Override public void createProducers(List topics) { Timer timer = new Timer(); diff --git a/driver-api/pom.xml b/driver-api/pom.xml index 85a7d7fb5..1e043bd50 100644 --- a/driver-api/pom.xml +++ b/driver-api/pom.xml @@ -31,5 +31,11 @@ bookkeeper-stats-api ${bookkeeper.version} + + org.projectlombok + lombok + 1.18.24 + provided + - \ No newline at end of file + diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java index 99d137312..b4a0ba9a9 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java @@ -15,7 +15,9 @@ import java.io.File; import java.io.IOException; +import java.util.List; import java.util.concurrent.CompletableFuture; +import lombok.Value; import org.apache.bookkeeper.stats.StatsLogger; /** @@ -44,6 +46,17 @@ public interface BenchmarkDriver extends AutoCloseable { */ CompletableFuture createTopic(String topic, int partitions); + /** + * Create a list of new topics with the given number of partitions + */ + default CompletableFuture createTopics(List topicInfos) { + @SuppressWarnings("unchecked") + CompletableFuture[] futures = topicInfos.stream() + .map(topicInfo -> createTopic(topicInfo.getTopic(), topicInfo.getPartitions())) + .toArray(CompletableFuture[]::new); + return CompletableFuture.allOf(futures); + } + /** * Create a producer for a given topic */ @@ -64,4 +77,10 @@ CompletableFuture createConsumer( String topic, String subscriptionName, ConsumerCallback consumerCallback); + + @Value + class TopicInfo { + String topic; + int partitions; + } } diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index e87b7d812..cc7d416be 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -36,6 +36,36 @@ kafka-clients 3.2.1 + + org.projectlombok + lombok + 1.18.24 + provided + + + org.assertj + assertj-core + 3.23.1 + test + + + org.junit.jupiter + junit-jupiter + 5.9.0 + test + + + org.mockito + mockito-junit-jupiter + 4.7.0 + test + + + org.slf4j + slf4j-simple + ${slf4j.version} + test + diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java index a87ffe8de..5bf03a95e 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java @@ -31,7 +31,6 @@ import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.DeleteTopicsResult; import org.apache.kafka.clients.admin.ListTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; @@ -107,20 +106,17 @@ public String getTopicNamePrefix() { return "test-topic"; } - @SuppressWarnings({ "rawtypes", "unchecked" }) @Override public CompletableFuture createTopic(String topic, int partitions) { - NewTopic newTopic = new NewTopic(topic, partitions, config.replicationFactor); - newTopic.configs(new HashMap<>((Map) topicProperties)); - CompletableFuture promise = new CompletableFuture<>(); - admin.createTopics(Collections.singletonList(newTopic)).all().whenComplete((__, e) -> { - if (e != null) { - promise.completeExceptionally(e); - } else { - promise.complete(null); - } - }); - return promise; + return createTopics(Collections.singletonList(new TopicInfo(topic, partitions))); + } + + @Override + public CompletableFuture createTopics(List topicInfos) { + @SuppressWarnings({"unchecked", "rawtypes"}) + Map topicConfigs = new HashMap<>((Map) topicProperties); + KafkaTopicCreator topicCreator = new KafkaTopicCreator(admin, topicConfigs, config.replicationFactor); + return topicCreator.create(topicInfos); } @Override diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java new file mode 100644 index 000000000..1b485eb66 --- /dev/null +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java @@ -0,0 +1,105 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.driver.kafka; + +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; +import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.errors.TopicExistsException; + +@Slf4j +@RequiredArgsConstructor +class KafkaTopicCreator { + private static final int MAX_BATCH_SIZE = 500; + private final AdminClient admin; + private final Map topicConfigs; + private final short replicationFactor; + private final int maxBatchSize; + + KafkaTopicCreator(AdminClient admin, Map topicConfigs, short replicationFactor) { + this(admin, topicConfigs, replicationFactor, MAX_BATCH_SIZE); + } + + CompletableFuture create(List topicInfos) { + return CompletableFuture.runAsync(() -> createBlocking(topicInfos)); + } + + private void createBlocking(List topicInfos) { + BlockingQueue queue = new ArrayBlockingQueue<>(topicInfos.size(), true, topicInfos); + List batch = new ArrayList<>(); + AtomicInteger succeeded = new AtomicInteger(); + while (succeeded.get() < topicInfos.size()) { + int batchSize = queue.drainTo(batch, maxBatchSize); + if (batchSize > 0) { + executeBatch(batch).forEach((topicInfo, success) -> { + if (success) { + succeeded.incrementAndGet(); + } else { + //noinspection ResultOfMethodCallIgnored + queue.offer(topicInfo); + } + }); + log.info("Created batch of {}", batchSize); + batch.clear(); + } + } + } + + private Map executeBatch(List batch) { + log.debug("Executing batch, size: {}", batch.size()); + Map lookup = batch.stream() + .collect(toMap(TopicInfo::getTopic, identity())); + + List newTopics = batch.stream() + .map(this::newTopic) + .collect(toList()); + + return admin.createTopics(newTopics).values() + .entrySet().stream() + .collect(toMap(e -> lookup.get(e.getKey()), e -> isSuccess(e.getValue()))); + } + + private NewTopic newTopic(TopicInfo topicInfo) { + NewTopic newTopic = new NewTopic(topicInfo.getTopic(), topicInfo.getPartitions(), replicationFactor); + newTopic.configs(topicConfigs); + return newTopic; + } + + private boolean isSuccess(KafkaFuture future) { + try { + future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (ExecutionException e) { + log.debug(e.getMessage()); + return e.getCause() instanceof TopicExistsException; + } + return true; + } +} diff --git a/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java b/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java new file mode 100644 index 000000000..034f2b3cb --- /dev/null +++ b/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java @@ -0,0 +1,124 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.driver.kafka; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeoutException; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.errors.TopicExistsException; +import org.apache.kafka.common.internals.KafkaFutureImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class KafkaTopicCreatorTest { + private final Map topicConfigs = new HashMap<>(); + private final String topic = "topic"; + private final int partitions = 1; + private final short replicationFactor = 1; + private final TopicInfo topicInfo = new TopicInfo(topic, partitions); + @Mock + private AdminClient admin; + @Mock + private CreateTopicsResult createTopicsResult; + @Captor + private ArgumentCaptor> captor; + private KafkaTopicCreator topicCreator; + + @BeforeEach + void beforeEach() { + int maxBatchSize = 1; + topicCreator = new KafkaTopicCreator(admin, topicConfigs, replicationFactor, maxBatchSize); + + when(admin.createTopics(any())).thenAnswer(__ -> createTopicsResult); + } + + @Test + void created() { + KafkaFuture future = KafkaFuture.completedFuture(null); + + when(createTopicsResult.values()).thenReturn(singletonMap(topic, future)); + + topicCreator.create(singletonList(topicInfo)).join(); + + verify(admin).createTopics(captor.capture()); + + List> allValues = captor.getAllValues(); + assertThat(allValues).hasSize(1); + assertNewTopics(allValues.get(0)); + } + + @Test + void topicExists() { + KafkaFutureImpl future = new KafkaFutureImpl<>(); + future.completeExceptionally(new TopicExistsException(null)); + + when(createTopicsResult.values()).thenReturn(singletonMap(topic, future)); + + topicCreator.create(singletonList(topicInfo)).join(); + + verify(admin).createTopics(captor.capture()); + + List> allValues = captor.getAllValues(); + assertThat(allValues).hasSize(1); + assertNewTopics(allValues.get(0)); + } + + @Test + void timeout() { + KafkaFutureImpl future1 = new KafkaFutureImpl<>(); + future1.completeExceptionally(new TimeoutException()); + KafkaFuture future2 = KafkaFuture.completedFuture(null); + + when(createTopicsResult.values()) + .thenReturn(singletonMap(topic, future1)) + .thenReturn(singletonMap(topic, future2)); + + topicCreator.create(singletonList(topicInfo)).join(); + + verify(admin, times(2)).createTopics(captor.capture()); + + List> allValues = captor.getAllValues(); + assertThat(allValues).hasSize(2); + assertNewTopics(allValues.get(0)); + assertNewTopics(allValues.get(1)); + } + + private void assertNewTopics(List newTopics) { + assertThat(newTopics).hasSize(1); + NewTopic newTopic = newTopics.get(0); + assertThat(newTopic.name()).isEqualTo(topic); + assertThat(newTopic.numPartitions()).isEqualTo(partitions); + assertThat(newTopic.replicationFactor()).isEqualTo(replicationFactor); + assertThat(newTopic.configs()).isSameAs(topicConfigs); + } +} \ No newline at end of file diff --git a/driver-kafka/src/test/resources/simplelogger.properties b/driver-kafka/src/test/resources/simplelogger.properties new file mode 100644 index 000000000..a38a4f465 --- /dev/null +++ b/driver-kafka/src/test/resources/simplelogger.properties @@ -0,0 +1,18 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.slf4j.simpleLogger.showDateTime=true +org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS +org.slf4j.simpleLogger.log.org.apache.kafka=WARN +org.slf4j.simpleLogger.log.io.openmessaging.benchmark.driver.kafka=DEBUG diff --git a/pom.xml b/pom.xml index e31822455..720e8d8ae 100644 --- a/pom.xml +++ b/pom.xml @@ -58,6 +58,7 @@ UTF-8 4.14.0 2.13.2 + 1.7.36 @@ -93,6 +94,10 @@ + + maven-surefire-plugin + 3.0.0-M7 + @@ -100,7 +105,7 @@ org.slf4j slf4j-api - 1.7.25 + ${slf4j.version} From bb2cdaf316d5b1549b5ddf97d3718298cf2eeb2c Mon Sep 17 00:00:00 2001 From: Elliot West Date: Wed, 14 Sep 2022 17:44:12 +0100 Subject: [PATCH 034/103] Update RMQ versions (#294) --- driver-rabbitmq/deploy/deploy.yaml | 10 ++++++---- driver-rabbitmq/deploy/terraform.tfvars | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/driver-rabbitmq/deploy/deploy.yaml b/driver-rabbitmq/deploy/deploy.yaml index e9ee13b70..27f35a595 100644 --- a/driver-rabbitmq/deploy/deploy.yaml +++ b/driver-rabbitmq/deploy/deploy.yaml @@ -43,8 +43,8 @@ connection: ssh tasks: - set_fact: - erlangVersion: 23.3.4.10 - rabbitMqVersion: 3.9.11 + erlangVersion: 24.3.4.5 + rabbitMqVersion: 3.10.7 - name: Install RPM packages yum: pkg={{ item }} state=latest with_items: @@ -54,12 +54,14 @@ - socat - name: Install Erlang yum: - name: https://github.com/rabbitmq/erlang-rpm/releases/download/v{{ erlangVersion }}/erlang-{{ erlangVersion }}-1.el7.x86_64.rpm + name: https://github.com/rabbitmq/erlang-rpm/releases/download/v{{ erlangVersion }}/erlang-{{ erlangVersion }}-1.el8.x86_64.rpm state: present + disable_gpg_check: yes - name: Install Rabbitmq Server yum: - name: https://github.com/rabbitmq/rabbitmq-server/releases/download/v{{ rabbitMqVersion }}/rabbitmq-server-{{ rabbitMqVersion }}-1.el7.noarch.rpm + name: https://github.com/rabbitmq/rabbitmq-server/releases/download/v{{ rabbitMqVersion }}/rabbitmq-server-{{ rabbitMqVersion }}-1.el8.noarch.rpm state: present + disable_gpg_check: yes - name: Create rabbitmq-env.conf file template: diff --git a/driver-rabbitmq/deploy/terraform.tfvars b/driver-rabbitmq/deploy/terraform.tfvars index 273acd4d1..d671bf691 100644 --- a/driver-rabbitmq/deploy/terraform.tfvars +++ b/driver-rabbitmq/deploy/terraform.tfvars @@ -1,6 +1,6 @@ public_key_path = "~/.ssh/rabbitmq_aws.pub" region = "us-west-2" -ami = "ami-9fa343e7" // RHEL-7.4 +ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { "rabbitmq" = "i3en.6xlarge" @@ -10,4 +10,4 @@ instance_types = { num_instances = { "rabbitmq" = 3 "client" = 4 -} \ No newline at end of file +} From 4d6f53f3f1ffa0aa1f0f63c820794ca1f754f6ec Mon Sep 17 00:00:00 2001 From: Elliot West Date: Wed, 14 Sep 2022 21:48:01 +0100 Subject: [PATCH 035/103] [RabbitMQ] Introduce AZ var, default to us-west-2a (#293) * Introduce AZ var, default to us-west-2a * Update documentation to mention az * Change HCL ref format. Update README with AZ * Update README with AZ --- driver-rabbitmq/README.md | 6 ++++-- driver-rabbitmq/deploy/provision-rabbitmq-aws.tf | 2 ++ driver-rabbitmq/deploy/terraform.tfvars | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/driver-rabbitmq/README.md b/driver-rabbitmq/README.md index 16b9b1400..daca66070 100644 --- a/driver-rabbitmq/README.md +++ b/driver-rabbitmq/README.md @@ -69,6 +69,7 @@ There's a handful of configurable parameters related to the Terraform deployment Variable | Description | Default :--------|:------------|:------- `region` | The AWS region in which the RabbitMQ cluster will be deployed | `us-west-2` +`az` | The availability zone in which the RabbitMQ cluster will be deployed | `us-west-2a` `public_key_path` | The path to the SSH public key that you've generated | `~/.ssh/rabbitmq_aws.pub` `ami` | The [Amazon Machine Image](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) (AWI) to be used by the cluster's machines | [`ami-9fa343e7`](https://access.redhat.com/articles/3135091) `instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (RabbitMQ brokers), `c4.8xlarge` (benchmarking client) @@ -77,10 +78,11 @@ Variable | Description | Default ### Running the Ansible playbook -With the appropriate infrastructure in place, you can install and start the RabbitMQ cluster using Ansible with just one command: +With the appropriate infrastructure in place, you can install and start the RabbitMQ cluster using Ansible with just one command. +Note that a `TFSTATE` environment must point to the folder in which the `tf.state` file is located. ```bash -$ ansible-playbook \ +$ TF_STATE=. ansible-playbook \ --user ec2-user \ --inventory `which terraform-inventory` \ deploy.yaml diff --git a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf index 08dce56cc..d6bc0112a 100644 --- a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf +++ b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf @@ -22,6 +22,7 @@ variable "num_instances" { } variable "region" {} +variable "az" {} variable "ami" {} provider "aws" { @@ -54,6 +55,7 @@ resource "aws_subnet" "benchmark_subnet" { vpc_id = "${aws_vpc.benchmark_vpc.id}" cidr_block = "10.0.0.0/24" map_public_ip_on_launch = true + availability_zone = "${var.az}" } resource "aws_security_group" "benchmark_security_group" { diff --git a/driver-rabbitmq/deploy/terraform.tfvars b/driver-rabbitmq/deploy/terraform.tfvars index d671bf691..79275c107 100644 --- a/driver-rabbitmq/deploy/terraform.tfvars +++ b/driver-rabbitmq/deploy/terraform.tfvars @@ -1,5 +1,6 @@ public_key_path = "~/.ssh/rabbitmq_aws.pub" region = "us-west-2" +az = "us-west-2a" ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { From 882a5bc4514e301f5a89e6ceafedd553731d6f61 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Wed, 14 Sep 2022 23:43:09 +0100 Subject: [PATCH 036/103] Improve visibility and stability of benchmark execution (#295) * Improve visibility and stability of benchmark execution ## Motivation As a new user running benchmarks sometimes it's not clear what's happening at any given moment. It's useful to add extra logging to give more feedback to the user. Additionally, the configuration could be more stable under larger workloads. ## Changes * During startup, to test everything is ready a single message is sent from each producer. The send and received counts are now logged every 2 seconds * Added rolling-file logging config to allow the user to ssh onto other workers to check the logs * Added ServerAliveInterval=60 to the ansible configuration so it's more stable during long running tasks * Added request.timeout.ms=1200000 to make Kafka API calls more stable until larger workloads * Added periodic logging of Kafka topic creation status * In DistributedWorkersEnsemble, the use of Pulsar's FutureUtil has been replaced with standard CompletableFuture methods * Update benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java Co-authored-by: Matteo Merli --- .../benchmark/WorkloadGenerator.java | 5 ++- .../worker/DistributedWorkersEnsemble.java | 40 +++++++++++-------- .../src/main/resources/log4j2.yaml | 17 ++++++-- .../deploy/ssd-deployment/ansible.cfg | 5 ++- driver-kafka/kafka-throughput.yaml | 3 +- .../driver/kafka/KafkaTopicCreator.java | 39 ++++++++++++------ 6 files changed, 71 insertions(+), 38 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 609737152..a8469808b 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -158,10 +158,11 @@ private void ensureTopicsAreReady() throws IOException { long end = start + 60 * 1000; while (System.currentTimeMillis() < end) { CountersStats stats = worker.getCountersStats(); - + + log.info("Waiting for topics to be ready -- Sent: {}, Received: {}", stats.messagesSent, stats.messagesReceived); if (stats.messagesReceived < expectedMessages) { try { - Thread.sleep(100); + Thread.sleep(2_000); } catch (InterruptedException e) { throw new RuntimeException(e); } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index 95ae70945..cdf90b2f9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -25,10 +25,10 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import java.util.zip.DataFormatException; import org.HdrHistogram.Histogram; -import org.apache.pulsar.common.util.FutureUtil; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.DefaultAsyncHttpClientConfig; import org.asynchttpclient.Dsl; @@ -71,12 +71,12 @@ public DistributedWorkersEnsemble(List workers, boolean extraConsumerWor httpClient = asyncHttpClient(clientBuilder); this.workers = workers; - // For driver-jms extra consumers are required. - // If there is an odd number of workers then allocate the extra to consumption. - int numberOfProducerWorkers = extraConsumerWorkers ? (workers.size() + 2) / 3 : workers.size() / 2; - List> partitions = Lists.partition(Lists.reverse(workers), workers.size() - numberOfProducerWorkers); - this.producerWorkers = partitions.get(1); - this.consumerWorkers = partitions.get(0); + // For driver-jms extra consumers are required. + // If there is an odd number of workers then allocate the extra to consumption. + int numberOfProducerWorkers = extraConsumerWorkers ? (workers.size() + 2) / 3 : workers.size() / 2; + List> partitions = Lists.partition(Lists.reverse(workers), workers.size() - numberOfProducerWorkers); + this.producerWorkers = partitions.get(1); + this.consumerWorkers = partitions.get(0); log.info("Workers list - producers: {}", producerWorkers); log.info("Workers list - consumers: {}", consumerWorkers); @@ -109,7 +109,7 @@ public void createProducers(List topics) { // Number of actually used workers might be less than available workers numberOfUsedProducerWorkers = i; - List> futures = topicsPerProducerMap.keySet().stream().map(producer -> { + CompletableFuture[] futures = topicsPerProducerMap.keySet().stream().map(producer -> { try { return sendPost(producer, "/create-producers", writer.writeValueAsBytes(topicsPerProducerMap.get(producer))); @@ -118,9 +118,9 @@ public void createProducers(List topics) { future.completeExceptionally(e); return future; } - }).collect(toList()); + }).toArray(this::newArray); - FutureUtil.waitForAll(futures).join(); + CompletableFuture.allOf(futures).join(); } @Override @@ -170,7 +170,7 @@ public void createConsumers(ConsumerAssignment overallConsumerAssignment) { topicsPerWorkerMap.put(consumerWorkers.get(i++), individualAssignement); } - List> futures = topicsPerWorkerMap.keySet().stream().map(consumer -> { + CompletableFuture[] futures = topicsPerWorkerMap.keySet().stream().map(consumer -> { try { return sendPost(consumer, "/create-consumers", writer.writeValueAsBytes(topicsPerWorkerMap.get(consumer))); @@ -179,9 +179,9 @@ public void createConsumers(ConsumerAssignment overallConsumerAssignment) { future.completeExceptionally(e); return future; } - }).collect(toList()); + }).toArray(this::newArray); - FutureUtil.waitForAll(futures).join(); + CompletableFuture.allOf(futures).join(); } @Override @@ -273,7 +273,8 @@ public void resetStats() throws IOException { * Send a request to multiple hosts and wait for all responses */ private void sendPost(List hosts, String path, byte[] body) { - FutureUtil.waitForAll(hosts.stream().map(w -> sendPost(w, path, body)).collect(toList())).join(); + CompletableFuture[] futures = hosts.stream().map(w -> sendPost(w, path, body)).toArray(this::newArray); + CompletableFuture.allOf(futures).join(); } private CompletableFuture sendPost(String host, String path, byte[] body) { @@ -287,11 +288,11 @@ private CompletableFuture sendPost(String host, String path, byte[] body) } private List get(List hosts, String path, Class clazz) { - List> futures = hosts.stream().map(w -> get(w, path, clazz)).collect(toList()); + CompletableFuture[] futures = hosts.stream().map(w -> get(w, path, clazz)).toArray(this::newArray); CompletableFuture> resultFuture = new CompletableFuture<>(); - FutureUtil.waitForAll(futures).thenRun(() -> { - resultFuture.complete(futures.stream().map(CompletableFuture::join).collect(toList())); + CompletableFuture.allOf(futures).thenRun(() -> { + resultFuture.complete(Stream.of(futures).map(CompletableFuture::join).collect(toList())); }).exceptionally(ex -> { resultFuture.completeExceptionally(ex); return null; @@ -333,6 +334,11 @@ public void close() throws Exception { httpClient.close(); } + @SuppressWarnings("unchecked") + private CompletableFuture[] newArray(int size) { + return new CompletableFuture[size]; + } + private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter(); private static final ObjectMapper mapper = new ObjectMapper() diff --git a/benchmark-framework/src/main/resources/log4j2.yaml b/benchmark-framework/src/main/resources/log4j2.yaml index 0c654be37..4579aeff2 100644 --- a/benchmark-framework/src/main/resources/log4j2.yaml +++ b/benchmark-framework/src/main/resources/log4j2.yaml @@ -22,12 +22,21 @@ Configuration: target: SYSTEM_OUT PatternLayout: Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level - %msg%n" - + RollingFile: + name: RollingFile + fileName: benchmark-worker.log + filePattern: benchmark-worker.log.%d{yyyy-MM-dd-hh-mm-ss}.gz + PatternLayout: + Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level - %msg%n" + Policies: + SizeBasedTriggeringPolicy: + size: 100MB + DefaultRollOverStrategy: + max: 10 Loggers: Root: level: info additivity: false AppenderRef: - ref: Console - - \ No newline at end of file + - ref: Console + - ref: RollingFile diff --git a/driver-kafka/deploy/ssd-deployment/ansible.cfg b/driver-kafka/deploy/ssd-deployment/ansible.cfg index d1fac48cd..0a5b72c32 100644 --- a/driver-kafka/deploy/ssd-deployment/ansible.cfg +++ b/driver-kafka/deploy/ssd-deployment/ansible.cfg @@ -5,4 +5,7 @@ private_key_file=~/.ssh/kafka_aws [privilege_escalation] become=true become_method='sudo' -become_user='root' \ No newline at end of file +become_user='root' + +[ssh_connection] +ssh_args=-o ServerAliveInterval=60 diff --git a/driver-kafka/kafka-throughput.yaml b/driver-kafka/kafka-throughput.yaml index 445202bd2..9370528e4 100644 --- a/driver-kafka/kafka-throughput.yaml +++ b/driver-kafka/kafka-throughput.yaml @@ -25,7 +25,8 @@ topicConfig: | commonConfig: | bootstrap.servers=localhost:9092 - default.api.timeout.ms=600000 + default.api.timeout.ms=1200000 + request.timeout.ms=1200000 producerConfig: | acks=all diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java index 1b485eb66..af1c949f5 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.kafka; +import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.function.Function.identity; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; @@ -24,6 +25,9 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicInteger; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -36,6 +40,7 @@ @RequiredArgsConstructor class KafkaTopicCreator { private static final int MAX_BATCH_SIZE = 500; + private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); private final AdminClient admin; private final Map topicConfigs; private final short replicationFactor; @@ -53,20 +58,28 @@ private void createBlocking(List topicInfos) { BlockingQueue queue = new ArrayBlockingQueue<>(topicInfos.size(), true, topicInfos); List batch = new ArrayList<>(); AtomicInteger succeeded = new AtomicInteger(); - while (succeeded.get() < topicInfos.size()) { - int batchSize = queue.drainTo(batch, maxBatchSize); - if (batchSize > 0) { - executeBatch(batch).forEach((topicInfo, success) -> { - if (success) { - succeeded.incrementAndGet(); - } else { - //noinspection ResultOfMethodCallIgnored - queue.offer(topicInfo); - } - }); - log.info("Created batch of {}", batchSize); - batch.clear(); + + ScheduledFuture loggingFuture = + executor.scheduleAtFixedRate(() -> log.info("Created topics {}/{}", succeeded.get(), topicInfos.size()), + 10, 10, SECONDS); + + try { + while (succeeded.get() < topicInfos.size()) { + int batchSize = queue.drainTo(batch, maxBatchSize); + if (batchSize > 0) { + executeBatch(batch).forEach((topicInfo, success) -> { + if (success) { + succeeded.incrementAndGet(); + } else { + //noinspection ResultOfMethodCallIgnored + queue.offer(topicInfo); + } + }); + batch.clear(); + } } + } finally { + loggingFuture.cancel(true); } } From d923ce8b358981375ca54351f68e2b7e9906e8be Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 15 Sep 2022 16:29:34 +0100 Subject: [PATCH 037/103] Add ansible tags to kafka (#297) ## Motivation Sometimes there's a need to update the benchmark code, configuration or workloads or restart Kafka brokers. It can be useful to be able to run a subset of the ansible tasks. ## Changes * Add client-code and restart-kafka tags to the Kafka ansible --- .../deploy/ssd-deployment/deploy.yaml | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/driver-kafka/deploy/ssd-deployment/deploy.yaml b/driver-kafka/deploy/ssd-deployment/deploy.yaml index 80a06392e..6a1e502bb 100644 --- a/driver-kafka/deploy/ssd-deployment/deploy.yaml +++ b/driver-kafka/deploy/ssd-deployment/deploy.yaml @@ -102,10 +102,11 @@ - file: path=/opt/kafka state=directory - set_fact: zookeeperServers: "{{ groups['zookeeper'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | map('regex_replace', '^(.*)$', '\\1:2181') | join(',') }}" - boostrapServers: "{{ groups['kafka'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:9092') | join(',') }}" + bootstrapServers: "{{ groups['kafka'] | map('extract', hostvars, ['private_ip']) | map('regex_replace', '^(.*)$', '\\1:9092') | join(',') }}" kafkaVersion: "3.2.1" + tags: [client-code] - debug: - msg: "zookeeper servers: {{ zookeeperServers }}\nboostrap servers: {{ boostrapServers }}" + msg: "zookeeper servers: {{ zookeeperServers }}\nbootstrap servers: {{ bootstrapServers }}" - name: Download Kafka package unarchive: src: "http://archive.apache.org/dist/kafka/{{ kafkaVersion }}/kafka_2.13-{{ kafkaVersion }}.tgz" @@ -158,6 +159,7 @@ state: restarted daemon_reload: yes name: "kafka" + tags: [restart-kafka] - name: Chrony setup hosts: client @@ -179,23 +181,28 @@ become: true tasks: - file: path=/opt/benchmark state=absent + tags: [client-code] - name: Copy benchmark code unarchive: src: ../../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz dest: /opt + tags: [client-code] - shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark + tags: [client-code] - shell: tuned-adm profile latency-performance - name: Get list of driver config files raw: ls -1 /opt/benchmark/driver-kafka/*.yaml register: drivers_list + tags: [client-code] - name: Configure Bootstrap Servers lineinfile: dest: '{{ item }}' regexp: '^ bootstrap.servers=' - line: ' bootstrap.servers={{ boostrapServers }}' + line: ' bootstrap.servers={{ bootstrapServers }}' with_items: '{{ drivers_list.stdout_lines }}' + tags: [client-code] - name: Get list of jms driver config files raw: ls -1 /opt/benchmark/driver-jms/kafka*.yaml @@ -205,14 +212,14 @@ lineinfile: dest: '{{ item }}' regexp: '^ bootstrap.servers=' - line: ' bootstrap.servers={{ boostrapServers }}' + line: ' bootstrap.servers={{ bootstrapServers }}' with_items: '{{ jms_drivers_list.stdout_lines }}' - name: Configure JMS Connection Factory ansible.builtin.replace: dest: '{{ item }}' regexp: 'localhost\:9092' - replace: '{{ boostrapServers }}' + replace: '{{ bootstrapServers }}' with_items: '{{ jms_drivers_list.stdout_lines }}' - name: Configure memory @@ -220,22 +227,27 @@ dest: /opt/benchmark/bin/benchmark-worker regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB"' + tags: [client-code] - name: Configure memory lineinfile: dest: /opt/benchmark/bin/benchmark regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xmx4G"' + tags: [client-code] - template: src: "templates/workers.yaml" dest: "/opt/benchmark/workers.yaml" + tags: [client-code] - name: Install benchmark systemd service template: src: "templates/benchmark-worker.service" dest: "/etc/systemd/system/benchmark-worker.service" + tags: [client-code] - systemd: state: restarted daemon_reload: yes name: "benchmark-worker" + tags: [client-code] - name: Hosts addresses hosts: localhost From f8122355d064c92335290ca4703628a74979dff7 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 15 Sep 2022 18:13:14 +0100 Subject: [PATCH 038/103] Improve Kafka infrastructure stability (#298) ## Motivation Cleanup Kafka to remove warning and ensure stability ## Changes * add Kafka ansible ssh retries * Replace '-' with '_' in Kafka ansible Name tags to remove warnings * Add additional tags to easily identify EC2 instances --- driver-kafka/deploy/ssd-deployment/ansible.cfg | 1 + .../deploy/ssd-deployment/provision-kafka-aws.tf | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/driver-kafka/deploy/ssd-deployment/ansible.cfg b/driver-kafka/deploy/ssd-deployment/ansible.cfg index 0a5b72c32..066d4f2e4 100644 --- a/driver-kafka/deploy/ssd-deployment/ansible.cfg +++ b/driver-kafka/deploy/ssd-deployment/ansible.cfg @@ -9,3 +9,4 @@ become_user='root' [ssh_connection] ssh_args=-o ServerAliveInterval=60 +retries=10 diff --git a/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf b/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf index e1f99472f..fb8f1f784 100644 --- a/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf +++ b/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf @@ -45,7 +45,7 @@ resource "aws_vpc" "benchmark_vpc" { cidr_block = "10.0.0.0/16" tags = { - Name = "Kafka-Benchmark-VPC-${random_id.hash.hex}" + Name = "Kafka_Benchmark_VPC_${random_id.hash.hex}" } } @@ -116,7 +116,8 @@ resource "aws_instance" "zookeeper" { count = "${var.num_instances["zookeeper"]}" tags = { - Name = "zk-${count.index}" + Name = "zk_${count.index}" + Benchmark = "Kafka" } } @@ -129,7 +130,8 @@ resource "aws_instance" "kafka" { count = "${var.num_instances["kafka"]}" tags = { - Name = "kafka-${count.index}" + Name = "kafka_${count.index}" + Benchmark = "Kafka" } } @@ -142,7 +144,8 @@ resource "aws_instance" "client" { count = "${var.num_instances["client"]}" tags = { - Name = "kafka-client-${count.index}" + Name = "kafka_client_${count.index}" + Benchmark = "Kafka" } } From 59cd951163934c21f5c08bc791993055cab3ec7c Mon Sep 17 00:00:00 2001 From: Elliot West Date: Thu, 15 Sep 2022 18:14:46 +0100 Subject: [PATCH 039/103] [Improvement] Consolidate project dependencies (#296) * Consolidate dependencies * Remove unnecessary relative paths * Clean up poms * newline --- benchmark-framework/pom.xml | 88 ++----------- driver-api/pom.xml | 23 +++- driver-artemis/pom.xml | 4 +- driver-bookkeeper/pom.xml | 10 +- driver-jms/pom.xml | 1 - .../driver/jms/JMSBenchmarkConsumer.java | 2 - .../driver/jms/JMSBenchmarkProducer.java | 2 - driver-kafka/pom.xml | 12 -- .../driver/kafka/KafkaBenchmarkProducer.java | 4 - driver-kop/pom.xml | 13 +- driver-nats/pom.xml | 2 +- .../driver/nats/NatsBenchmarkDriver.java | 1 - .../driver/nsq/NsqBenchmarkDriver.java | 1 - driver-pravega/pom.xml | 20 +-- driver-pulsar/pom.xml | 3 - driver-rabbitmq/pom.xml | 7 +- driver-redis/pom.xml | 6 +- driver-rocketmq/pom.xml | 3 - package/pom.xml | 3 +- pom.xml | 116 +++++++++++++----- 20 files changed, 136 insertions(+), 185 deletions(-) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index fabb7669b..b09311aee 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -20,14 +20,12 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. benchmark-framework - 2.17.1 - 2.13.2 + 9.4.42.v20210604 @@ -36,180 +34,110 @@ driver-api ${project.version} - org.hdrhistogram HdrHistogram 2.1.10 - - - com.google.guava - guava - - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-base - ${jackson.version} - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson.version} - - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - com.beust jcommander - 1.48 - ${project.groupId} driver-pulsar ${project.version} - ${project.groupId} driver-jms ${project.version} - ${project.groupId} driver-kafka ${project.version} - ${project.groupId} driver-kop ${project.version} - ${project.groupId} driver-pravega ${project.version} - ${project.groupId} driver-rocketmq ${project.version} - ${project.groupId} driver-rabbitmq ${project.version} - ${project.groupId} driver-artemis ${project.version} - ${project.groupId} driver-bookkeeper ${project.version} - ${project.groupId} driver-nats ${project.version} - ${project.groupId} driver-nats-streaming ${project.version} - ${project.groupId} driver-nsq ${project.version} - ${project.groupId} driver-redis ${project.version} - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${bookkeeper.version} - - org.apache.bookkeeper.stats prometheus-metrics-provider ${bookkeeper.version} - io.javalin javalin 1.3.0 - org.asynchttpclient async-http-client - 2.10.4 + 2.12.3 - org.eclipse.jetty jetty-server - 9.4.42.v20210604 + ${jetty.version} - - - junit - junit - 4.13.1 - test - - org.eclipse.jetty jetty-util - 9.4.42.v20210604 + ${jetty.version} + + + junit + junit - diff --git a/driver-api/pom.xml b/driver-api/pom.xml index 1e043bd50..e5a635e2d 100644 --- a/driver-api/pom.xml +++ b/driver-api/pom.xml @@ -20,12 +20,27 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. driver-api + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-base + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + org.apache.bookkeeper.stats bookkeeper-stats-api @@ -34,8 +49,10 @@ org.projectlombok lombok - 1.18.24 - provided + + + org.slf4j + slf4j-simple diff --git a/driver-artemis/pom.xml b/driver-artemis/pom.xml index 2427c466d..9c1f9989a 100644 --- a/driver-artemis/pom.xml +++ b/driver-artemis/pom.xml @@ -20,7 +20,6 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. driver-artemis @@ -32,11 +31,10 @@ driver-api ${project.version} - org.apache.activemq artemis-core-client - 2.4.0 + 2.23.1 diff --git a/driver-bookkeeper/pom.xml b/driver-bookkeeper/pom.xml index 119e1cc1d..c24790138 100644 --- a/driver-bookkeeper/pom.xml +++ b/driver-bookkeeper/pom.xml @@ -20,23 +20,23 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. driver-bookkeeper - - ${project.groupId} driver-api ${project.version} - org.apache.distributedlog distributedlog-core-shaded - 4.7.0 + 4.14.4 + + + io.netty + netty-all diff --git a/driver-jms/pom.xml b/driver-jms/pom.xml index 8f71eff58..b8cac2434 100644 --- a/driver-jms/pom.xml +++ b/driver-jms/pom.xml @@ -20,7 +20,6 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. driver-jms diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java index 44f6becc5..157a2656b 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java @@ -15,8 +15,6 @@ import javax.jms.BytesMessage; import javax.jms.Connection; -import javax.jms.JMSConsumer; -import javax.jms.JMSContext; import javax.jms.Message; import javax.jms.MessageConsumer; import javax.jms.Session; diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java index b24f0313d..d96533061 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java @@ -21,9 +21,7 @@ import javax.jms.BytesMessage; import javax.jms.CompletionListener; import javax.jms.Destination; -import javax.jms.JMSContext; import javax.jms.JMSException; -import javax.jms.JMSProducer; import javax.jms.Message; import javax.jms.MessageProducer; import javax.jms.Session; diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index cc7d416be..0da9c2af0 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -20,7 +20,6 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. driver-kafka @@ -30,7 +29,6 @@ driver-api ${project.version} - org.apache.kafka kafka-clients @@ -39,14 +37,10 @@ org.projectlombok lombok - 1.18.24 - provided org.assertj assertj-core - 3.23.1 - test org.junit.jupiter @@ -60,12 +54,6 @@ 4.7.0 test - - org.slf4j - slf4j-simple - ${slf4j.version} - test - diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java index f242bc2af..bcd80f5bc 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java @@ -26,13 +26,9 @@ */ package io.openmessaging.benchmark.driver.kafka; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; -import java.util.Random; import java.util.concurrent.CompletableFuture; -import org.apache.commons.lang.ArrayUtils; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; diff --git a/driver-kop/pom.xml b/driver-kop/pom.xml index 6090951f8..5770fdfe2 100644 --- a/driver-kop/pom.xml +++ b/driver-kop/pom.xml @@ -20,7 +20,6 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. driver-kop @@ -31,29 +30,29 @@ driver-api ${project.version} - ${project.groupId} driver-kafka ${project.version} - ${project.groupId} driver-pulsar ${project.version} - io.streamnative.pulsar.handlers kafka-payload-processor - 2.9.1.5 + 2.10.1.7 + + + org.apache.commons + commons-lang3 - org.testng testng - 7.4.0 + 7.6.1 test diff --git a/driver-nats/pom.xml b/driver-nats/pom.xml index 855ee7263..6518eae9c 100644 --- a/driver-nats/pom.xml +++ b/driver-nats/pom.xml @@ -28,7 +28,7 @@ io.nats jnats - 2.0.0 + 2.15.6 ${project.groupId} diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index 5f1693057..8c40bf632 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -19,7 +19,6 @@ import io.nats.client.Dispatcher; import io.nats.client.Nats; import io.nats.client.Options; -import java.nio.charset.StandardCharsets; import java.time.Duration; import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.BenchmarkDriver; diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java index e0e0d0a88..617619242 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.logging.log4j.LogManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/driver-pravega/pom.xml b/driver-pravega/pom.xml index dea4a10ce..6c8071f3a 100644 --- a/driver-pravega/pom.xml +++ b/driver-pravega/pom.xml @@ -20,14 +20,13 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. driver-pravega - 0.10.1 + 0.10.2 @@ -47,48 +46,39 @@ driver-api ${project.version} - io.pravega pravega-client ${pravegaVersion} - io.pravega pravega-keycloak-client - 0.9.0 + 0.11.0 - io.grpc grpc-all - 1.36.0 + 1.49.0 - io.perfmark perfmark-api - 0.23.0 + 0.25.0 compile - io.netty netty-all - 4.1.65.Final - com.google.protobuf protobuf-java-util - 3.4.0 + 3.21.5 - org.apache.commons commons-lang3 - 3.7 diff --git a/driver-pulsar/pom.xml b/driver-pulsar/pom.xml index 839ebf811..384f8023c 100644 --- a/driver-pulsar/pom.xml +++ b/driver-pulsar/pom.xml @@ -20,7 +20,6 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. driver-pulsar @@ -34,13 +33,11 @@ driver-api ${project.version} - org.apache.pulsar pulsar-client-all ${pulsar.version} - com.google.guava guava diff --git a/driver-rabbitmq/pom.xml b/driver-rabbitmq/pom.xml index 1828f94da..04b95ad32 100644 --- a/driver-rabbitmq/pom.xml +++ b/driver-rabbitmq/pom.xml @@ -29,11 +29,14 @@ driver-api ${project.version} - com.rabbitmq amqp-client 4.8.0 + + io.netty + netty-all + - \ No newline at end of file + diff --git a/driver-redis/pom.xml b/driver-redis/pom.xml index 2b3e68361..95e079b07 100644 --- a/driver-redis/pom.xml +++ b/driver-redis/pom.xml @@ -31,19 +31,15 @@ driver-api ${project.version} - redis.clients jedis - 3.6.0 + 3.7.0 - com.google.guava guava - - diff --git a/driver-rocketmq/pom.xml b/driver-rocketmq/pom.xml index bc812c6bf..6fe1abbb6 100644 --- a/driver-rocketmq/pom.xml +++ b/driver-rocketmq/pom.xml @@ -27,7 +27,6 @@ 4.9.3 - 29.0-jre @@ -36,7 +35,6 @@ driver-api ${project.version} - org.apache.rocketmq rocketmq-acl @@ -55,7 +53,6 @@ com.google.guava guava - ${guava.version} diff --git a/package/pom.xml b/package/pom.xml index 07235dfba..5af96eb4c 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -21,7 +21,6 @@ io.openmessaging.benchmark messaging-benchmark 0.0.1-SNAPSHOT - .. package @@ -32,7 +31,7 @@ maven-assembly-plugin - 3.1.0 + 3.3.0 distro-assembly diff --git a/pom.xml b/pom.xml index 720e8d8ae..58cd8d4b6 100644 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ package docker driver-kop - + @@ -56,8 +56,16 @@ UTF-8 UTF-8 - 4.14.0 + + 3.23.1 + 4.14.4 + 3.12.0 + 2.17.1 + 1.18.24 2.13.2 + 1.48 + 4.13.1 + 4.1.65.Final 1.7.36 @@ -101,39 +109,81 @@ - - - org.slf4j - slf4j-api - ${slf4j.version} - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson.version} - - - io.netty - netty-all - 4.1.48.Final - - - - - - com.google.guava - guava - 29.0-jre - + + com.beust + jcommander + ${jcommander.version} + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-base + ${jackson.version} + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + ${jackson.version} + + + com.google.guava + guava + 29.0-jre + + + io.netty + netty-all + ${netty.version} + + + junit + junit + ${junit.version} + test + + + org.assertj + assertj-core + ${assertj.version} + test + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.projectlombok + lombok + ${lombok.version} + provided + + + org.slf4j + slf4j-simple + ${slf4j.version} + From 13de8a60cebb6364f7acdce3385cd6bc185e8d7d Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 15 Sep 2022 18:16:16 +0100 Subject: [PATCH 040/103] Add a shutdownhook to stop all workers (#299) ## Motivation If one needs to prematurely terminate a benchmark workload, for example with a Ctrl-C, the workers will no stop. It would be useful to be able to stop the whole workload. ## Changes * Add a shutdown hook to call the /stop-all endpoint on all workers --- .../src/main/java/io/openmessaging/benchmark/Benchmark.java | 1 + .../java/io/openmessaging/benchmark/worker/LocalWorker.java | 2 +- .../src/main/java/io/openmessaging/benchmark/worker/Worker.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index 8d455911f..b9ac38603 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -128,6 +128,7 @@ public static void main(String[] args) throws Exception { if (arguments.workers != null && !arguments.workers.isEmpty()) { worker = new DistributedWorkersEnsemble(arguments.workers, arguments.extraConsumers); + Runtime.getRuntime().addShutdownHook(new Thread(worker::stopAll)); } else { // Use local worker implementation worker = new LocalWorker(); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index 3e1c37c1a..6180414a4 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -355,7 +355,7 @@ public void resetStats() throws IOException { } @Override - public void stopAll() throws IOException { + public void stopAll() { testCompleted = true; consumersArePaused = false; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java index 6331c6b79..57181b9d4 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java @@ -52,5 +52,5 @@ public interface Worker extends AutoCloseable { void resetStats() throws IOException; - void stopAll() throws IOException; + void stopAll(); } From d4933541a0b289db2fa4abf1ee68a46afd5b7bd7 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 15 Sep 2022 18:17:33 +0100 Subject: [PATCH 041/103] Remove Kafka reset option (#300) ## Motivation The Kafka driver has the ability to `reset` on initialisation which will delete all topics. All workers will attempt to perform this action which can result in contention for resources and UnknownTopicOrPartitionException. This can make it hard to execute workloads, particularly those with lots of topics. ## Changes * Remove the Kafka `reset` option * Expose the Kafka ssh IP address in the terraform to make it easier for users to ssh in to a broker and manually delete topics if needed with the following command `/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --delete --topic test-topic-.*` * Reorder the Kafka topic names to make selection deletion easier --- .../benchmark/worker/LocalWorker.java | 2 +- .../ssd-deployment/provision-kafka-aws.tf | 4 ++++ driver-kafka/kafka-big-batches-gzip.yaml | 1 - driver-kafka/kafka-big-batches-lz4.yaml | 1 - driver-kafka/kafka-big-batches-snappy.yaml | 1 - driver-kafka/kafka-big-batches-zstd.yaml | 1 - driver-kafka/kafka-big-batches.yaml | 1 - driver-kafka/kafka-compression-gzip.yaml | 1 - driver-kafka/kafka-compression-lz4.yaml | 1 - driver-kafka/kafka-compression-snappy.yaml | 1 - driver-kafka/kafka-compression-zstd.yaml | 1 - driver-kafka/kafka-exactly-once.yaml | 1 - driver-kafka/kafka-latency.yaml | 1 - driver-kafka/kafka-no-linger.yaml | 1 - driver-kafka/kafka-sync.yaml | 1 - driver-kafka/kafka-throughput.yaml | 1 - .../benchmark/driver/kafka/Config.java | 2 -- .../driver/kafka/KafkaBenchmarkDriver.java | 18 ------------------ 18 files changed, 5 insertions(+), 35 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index 6180414a4..bb1a95713 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -159,7 +159,7 @@ public List createTopics(TopicsInfo topicsInfo) { } private String generateTopicName(int i) { - return String.format("%s-%s-%07d", benchmarkDriver.getTopicNamePrefix(), RandomGenerator.getRandomString(), i); + return String.format("%s-%07d-%s", benchmarkDriver.getTopicNamePrefix(), i, RandomGenerator.getRandomString()); } @Override diff --git a/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf b/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf index fb8f1f784..172c01ee8 100644 --- a/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf +++ b/driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf @@ -149,6 +149,10 @@ resource "aws_instance" "client" { } } +output "kafka_ssh_host" { + value = "${aws_instance.kafka.0.public_ip}" +} + output "client_ssh_host" { value = "${aws_instance.client.0.public_ip}" } diff --git a/driver-kafka/kafka-big-batches-gzip.yaml b/driver-kafka/kafka-big-batches-gzip.yaml index 7bb353d1d..86e48eac4 100644 --- a/driver-kafka/kafka-big-batches-gzip.yaml +++ b/driver-kafka/kafka-big-batches-gzip.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-big-batches-lz4.yaml b/driver-kafka/kafka-big-batches-lz4.yaml index 19d7f2809..18453137d 100644 --- a/driver-kafka/kafka-big-batches-lz4.yaml +++ b/driver-kafka/kafka-big-batches-lz4.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-big-batches-snappy.yaml b/driver-kafka/kafka-big-batches-snappy.yaml index 0b9d0f777..ffd1d0990 100644 --- a/driver-kafka/kafka-big-batches-snappy.yaml +++ b/driver-kafka/kafka-big-batches-snappy.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-big-batches-zstd.yaml b/driver-kafka/kafka-big-batches-zstd.yaml index b7e173d6a..3a15a5e26 100644 --- a/driver-kafka/kafka-big-batches-zstd.yaml +++ b/driver-kafka/kafka-big-batches-zstd.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-big-batches.yaml b/driver-kafka/kafka-big-batches.yaml index 066c81cb9..b0c8638a5 100644 --- a/driver-kafka/kafka-big-batches.yaml +++ b/driver-kafka/kafka-big-batches.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-compression-gzip.yaml b/driver-kafka/kafka-compression-gzip.yaml index fe6d92f0a..011a98f4a 100644 --- a/driver-kafka/kafka-compression-gzip.yaml +++ b/driver-kafka/kafka-compression-gzip.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-compression-lz4.yaml b/driver-kafka/kafka-compression-lz4.yaml index 1534bf1ef..8fdccc982 100644 --- a/driver-kafka/kafka-compression-lz4.yaml +++ b/driver-kafka/kafka-compression-lz4.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-compression-snappy.yaml b/driver-kafka/kafka-compression-snappy.yaml index 40367ea2a..ae229768b 100644 --- a/driver-kafka/kafka-compression-snappy.yaml +++ b/driver-kafka/kafka-compression-snappy.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-compression-zstd.yaml b/driver-kafka/kafka-compression-zstd.yaml index 87581bda9..d769bbb5f 100644 --- a/driver-kafka/kafka-compression-zstd.yaml +++ b/driver-kafka/kafka-compression-zstd.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-exactly-once.yaml b/driver-kafka/kafka-exactly-once.yaml index b65ffbfcc..8a6ec7706 100644 --- a/driver-kafka/kafka-exactly-once.yaml +++ b/driver-kafka/kafka-exactly-once.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-latency.yaml b/driver-kafka/kafka-latency.yaml index 1da1cbf35..ed0d5a600 100644 --- a/driver-kafka/kafka-latency.yaml +++ b/driver-kafka/kafka-latency.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-no-linger.yaml b/driver-kafka/kafka-no-linger.yaml index e9868b351..aaccf8edb 100644 --- a/driver-kafka/kafka-no-linger.yaml +++ b/driver-kafka/kafka-no-linger.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-sync.yaml b/driver-kafka/kafka-sync.yaml index 0de4d795d..357b570c4 100644 --- a/driver-kafka/kafka-sync.yaml +++ b/driver-kafka/kafka-sync.yaml @@ -19,7 +19,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/kafka-throughput.yaml b/driver-kafka/kafka-throughput.yaml index 9370528e4..01122b46d 100644 --- a/driver-kafka/kafka-throughput.yaml +++ b/driver-kafka/kafka-throughput.yaml @@ -18,7 +18,6 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver # Kafka client-specific configuration replicationFactor: 3 -reset: true topicConfig: | min.insync.replicas=2 diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java index 7f28aaf57..59e2a96dd 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java @@ -23,6 +23,4 @@ public class Config { public String producerConfig; public String consumerConfig; - - public boolean reset; } diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java index 5bf03a95e..6892e80dc 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java @@ -23,14 +23,10 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.DeleteTopicsResult; -import org.apache.kafka.clients.admin.ListTopicsResult; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; @@ -85,20 +81,6 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I topicProperties.load(new StringReader(config.topicConfig)); admin = AdminClient.create(commonProperties); - - if (config.reset) { - // List existing topics - ListTopicsResult result = admin.listTopics(); - try { - Set topics = result.names().get(); - // Delete all existing topics - DeleteTopicsResult deletes = admin.deleteTopics(topics); - deletes.all().get(); - } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); - throw new IOException(e); - } - } } @Override From 2f55db627dccba191b565084781ae2fd851318c8 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 15 Sep 2022 19:45:07 +0100 Subject: [PATCH 042/103] Remove catch IOException. Follow up to #299 (#301) --- .../src/main/java/io/openmessaging/benchmark/Benchmark.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index b9ac38603..674ac3436 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -164,10 +164,7 @@ public static void main(String[] args) throws Exception { } catch (Exception e) { log.error("Failed to run the workload '{}' for driver '{}'", workload.name, driverConfig, e); } finally { - try { - worker.stopAll(); - } catch (IOException e) { - } + worker.stopAll(); } }); }); From 44371daf939c74602b39dbc583faa8ab512de089 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Fri, 16 Sep 2022 16:25:48 +0100 Subject: [PATCH 043/103] [Feature] Command Line Utility to create workloads from a template (#302) * Add moduele * Javadoc * Test * Remove import * Demo name format in template * README * Formatting --- pom.xml | 1 + tool/README.md | 63 ++++++++++++ tool/pom.xml | 60 ++++++++++++ .../tool/workload/WorkloadGenerationTool.java | 88 +++++++++++++++++ .../tool/workload/WorkloadGenerator.java | 97 +++++++++++++++++++ .../tool/workload/WorkloadNameFormat.java | 71 ++++++++++++++ .../tool/workload/WorkloadSetTemplate.java | 53 ++++++++++ .../tool/workload/WorkloadNameFormatTest.java | 46 +++++++++ tool/src/test/resources/template.yaml | 25 +++++ 9 files changed, 504 insertions(+) create mode 100644 tool/README.md create mode 100644 tool/pom.xml create mode 100644 tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerationTool.java create mode 100644 tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerator.java create mode 100644 tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormat.java create mode 100644 tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java create mode 100644 tool/src/test/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormatTest.java create mode 100644 tool/src/test/resources/template.yaml diff --git a/pom.xml b/pom.xml index 58cd8d4b6..2a12440c1 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,7 @@ package docker driver-kop + tool diff --git a/tool/README.md b/tool/README.md new file mode 100644 index 000000000..9acbbfc92 --- /dev/null +++ b/tool/README.md @@ -0,0 +1,63 @@ +# Tool + +## `WorkloadGenerationTool` + +Generates a set of `Workload` definition files from a `WorkloadSetTemplate` file. + +### Example + +Template: +```yaml +nameFormat: "${topics}-topics-${partitionsPerTopic}-partitions-${messageSize}b-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}" +topics: [1] +partitionsPerTopic: [1] +messageSize: [10000] +payloadFile: "payload/payload-100b.data" +subscriptionsPerTopic: [1] +consumerPerSubscription: [1, 2, 4, 8, 16, 32, 64] +producersPerTopic: [1, 2, 4, 8, 16, 32, 64] +producerRate: [50000] +consumerBacklogSizeGB: 0 +testDurationMinutes: 15 +``` + +Usage: +``` +mkdir my-workloads + io.openmessaging.benchmark.tool.workload.WorkloadGenerationTool \ + --templateFile=template.yaml \ + --outputFolder=my-workloads +``` + +Output: +``` +Starting benchmark with config: templateFile: "template.yaml" +outputFolder: "my-workloads" + +Generated: 1-topics-1-partitions-10kb-1p-1c-50k +Generated: 1-topics-1-partitions-10kb-1p-2c-50k +Generated: 1-topics-1-partitions-10kb-1p-4c-50k +... +Generated: 1-topics-1-partitions-10kb-64p-64c-50k +Generated 49 workloads. +``` + +Example generated workload: +```yaml +name: "1-topics-1-partitions-10kb-64p-2c-50k" +topics: 1 +partitionsPerTopic: 1 +keyDistributor: "NO_KEY" +messageSize: 10000 +useRandomizedPayloads: false +randomBytesRatio: 0.0 +randomizedPayloadPoolSize: 0 +payloadFile: "payload/payload-100b.data" +subscriptionsPerTopic: 1 +producersPerTopic: 64 +consumerPerSubscription: 2 +producerRate: 50000 +consumerBacklogSizeGB: 0 +testDurationMinutes: 5 +warmupDurationMinutes: 1 +``` \ No newline at end of file diff --git a/tool/pom.xml b/tool/pom.xml new file mode 100644 index 000000000..bd3f9ce24 --- /dev/null +++ b/tool/pom.xml @@ -0,0 +1,60 @@ + + + + messaging-benchmark + io.openmessaging.benchmark + 0.0.1-SNAPSHOT + + 4.0.0 + + tool + + + + io.openmessaging.benchmark + benchmark-framework + ${project.version} + + + org.projectlombok + lombok + + + com.beust + jcommander + + + org.slf4j + slf4j-simple + + + org.apache.commons + commons-text + 1.9 + + + org.apache.commons + commons-io + 1.3.2 + + + org.assertj + assertj-core + + + org.junit.jupiter + junit-jupiter + 5.9.0 + test + + + org.mockito + mockito-junit-jupiter + 4.7.0 + test + + + + \ No newline at end of file diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerationTool.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerationTool.java new file mode 100644 index 000000000..973636445 --- /dev/null +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerationTool.java @@ -0,0 +1,88 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.tool.workload; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import io.openmessaging.benchmark.Workload; +import java.io.File; +import java.io.IOException; +import java.util.List; +import lombok.extern.slf4j.Slf4j; + +/** + * Generates a set of {@link Workload} definition files from a {@link WorkloadSetTemplate} file. + */ +@Slf4j +public class WorkloadGenerationTool { + + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory().configure(YAMLGenerator.Feature.WRITE_DOC_START_MARKER, false)) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + static { + mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE); + } + + public static void main(String[] args) throws IOException { + final WorkloadGenerationTool.Arguments arguments = new WorkloadGenerationTool.Arguments(); + JCommander jc = new JCommander(arguments); + jc.setProgramName("workload-generator"); + + try { + jc.parse(args); + } catch (ParameterException e) { + System.err.println(e.getMessage()); + jc.usage(); + System.exit(-1); + } + + if (arguments.help) { + jc.usage(); + System.exit(-1); + } + + // Dump configuration variables + log.info("Starting benchmark with config: {}", mapper.writeValueAsString(arguments)); + + WorkloadSetTemplate template = mapper.readValue(arguments.templateFile, WorkloadSetTemplate.class); + List workloads = new WorkloadGenerator(template).generate(); + for (Workload w : workloads) { + File outputFile = null; + try { + outputFile = new File(arguments.outputFolder, w.name + ".yaml"); + mapper.writeValue(outputFile, w); + } catch (IOException e) { + log.error("Could not write file: {}", outputFile, e); + } + } + } + + static class Arguments { + @Parameter(names = {"-t", "--template-file"}, + description = "Path to a YAML file containing the workload template", required = true) + public File templateFile; + + @Parameter(names = {"-o", "--output-folder"}, description = "Output", required = true) + public File outputFolder; + + @Parameter(names = {"-h", "--help"}, description = "Help message", help = true) + boolean help; + } +} diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerator.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerator.java new file mode 100644 index 000000000..e81fb1668 --- /dev/null +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerator.java @@ -0,0 +1,97 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.tool.workload; + +import static java.util.Collections.unmodifiableList; +import io.openmessaging.benchmark.Workload; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +/** + * Expands a {@link io.openmessaging.benchmark.tool.workload.WorkloadSetTemplate} into a set of + * {@link io.openmessaging.benchmark.Workload Workloads}. + */ +@Slf4j +class WorkloadGenerator { + private final WorkloadSetTemplate template; + private final WorkloadNameFormat nameFormat; + + WorkloadGenerator(@NonNull WorkloadSetTemplate template) { + this.template = template; + nameFormat = new WorkloadNameFormat(template.nameFormat); + } + + List generate() throws IOException { + List workloads = new ArrayList<>(); + Workload workload = new Workload(); + workload.keyDistributor = template.keyDistributor; + workload.payloadFile = template.payloadFile; + workload.randomBytesRatio = template.randomBytesRatio; + workload.randomizedPayloadPoolSize = template.randomizedPayloadPoolSize; + workload.consumerBacklogSizeGB = template.consumerBacklogSizeGB; + workload.testDurationMinutes = template.testDurationMinutes; + workload.warmupDurationMinutes = template.warmupDurationMinutes; + workload.useRandomizedPayloads = template.useRandomizedPayloads; + for (int t : template.topics) { + for (int pa : template.partitionsPerTopic) { + for (int ms : template.messageSize) { + for (int pd : template.producersPerTopic) { + for (int st : template.subscriptionsPerTopic) { + for (int cn : template.consumerPerSubscription) { + for (int pr : template.producerRate) { + workload.topics = t; + workload.partitionsPerTopic = pa; + workload.messageSize = ms; + workload.producersPerTopic = pd; + workload.subscriptionsPerTopic = st; + workload.consumerPerSubscription = cn; + workload.producerRate = pr; + Workload copy = copyOf(workload); + workloads.add(copy); + log.info("Generated: {}", copy.name); + } + } + } + } + } + } + } + log.info("Generated {} workloads.", workloads.size()); + return unmodifiableList(workloads); + } + + private Workload copyOf(Workload workload) { + Workload copy = new Workload(); + copy.keyDistributor = workload.keyDistributor; + copy.payloadFile = workload.payloadFile; + copy.randomBytesRatio = workload.randomBytesRatio; + copy.randomizedPayloadPoolSize = workload.randomizedPayloadPoolSize; + copy.consumerBacklogSizeGB = workload.consumerBacklogSizeGB; + copy.testDurationMinutes = workload.testDurationMinutes; + copy.warmupDurationMinutes = workload.warmupDurationMinutes; + copy.topics = workload.topics; + copy.partitionsPerTopic = workload.partitionsPerTopic; + copy.messageSize = workload.messageSize; + copy.producersPerTopic = workload.producersPerTopic; + copy.subscriptionsPerTopic = workload.subscriptionsPerTopic; + copy.consumerPerSubscription = workload.consumerPerSubscription; + copy.producerRate = workload.producerRate; + copy.useRandomizedPayloads = workload.useRandomizedPayloads; + copy.name = nameFormat.from(copy); + return copy; + } +} diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormat.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormat.java new file mode 100644 index 000000000..d6a5f5218 --- /dev/null +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormat.java @@ -0,0 +1,71 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.tool.workload; + +import io.openmessaging.benchmark.Workload; +import java.util.HashMap; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang.text.StrSubstitutor; + +/** + * Generates {@link Workload} names based on a template. Substitutes template place-holders of the form + * {@code ${variableName}}, where {@code variableName} is the name of a public member in {@link Workload}. Note that the + * set of variables is statically assigned. Numeric values will typically be in a form that includes an SI suffix. + */ +@RequiredArgsConstructor +class WorkloadNameFormat { + + private static final long MAX_PRODUCER_RATE = 10_000_000; + + private final String format; + + String from(Workload workload) { + if (workload.name != null) { + return workload.name; + } + Map params = new HashMap<>(); + params.put("topics", countToDisplaySize(workload.topics)); + params.put("partitionsPerTopic", countToDisplaySize(workload.partitionsPerTopic)); + params.put("messageSize", countToDisplaySize(workload.messageSize)); + params.put("subscriptionsPerTopic", countToDisplaySize(workload.subscriptionsPerTopic)); + params.put("producersPerTopic", countToDisplaySize(workload.producersPerTopic)); + params.put("consumerPerSubscription", countToDisplaySize(workload.consumerPerSubscription)); + params.put("producerRate", + (workload.producerRate >= MAX_PRODUCER_RATE) ? "max-rate" : countToDisplaySize(workload.producerRate)); + params.put("keyDistributor", workload.keyDistributor); + params.put("payloadFile", workload.payloadFile); + params.put("useRandomizedPayloads", workload.useRandomizedPayloads); + params.put("randomBytesRatio", workload.randomBytesRatio); + params.put("randomizedPayloadPoolSize", countToDisplaySize(workload.randomizedPayloadPoolSize)); + params.put("consumerBacklogSizeGB", countToDisplaySize(workload.consumerBacklogSizeGB)); + params.put("testDurationMinutes", workload.testDurationMinutes); + params.put("warmupDurationMinutes", workload.warmupDurationMinutes); + return StrSubstitutor.replace(format, params, "${", "}"); + } + + private static String countToDisplaySize(long size) { + String displaySize; + if (size / 1_000_000_000L > 0L) { + displaySize = size / 1_000_000_000L + "g"; + } else if (size / 1_000_000L > 0L) { + displaySize = size / 1_000_000L + "m"; + } else if (size / 1_000L > 0L) { + displaySize = size / 1_000 + "k"; + } else { + displaySize = size + ""; + } + return displaySize; + } +} diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java new file mode 100644 index 000000000..32c588cf3 --- /dev/null +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java @@ -0,0 +1,53 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.tool.workload; + +import io.openmessaging.benchmark.utils.distributor.KeyDistributorType; +import java.util.Collections; +import java.util.List; +import lombok.NoArgsConstructor; +import lombok.Value; + +/** + * A template that defines a set of workload definitions. This is much like the + * {@link io.openmessaging.benchmark.Workload} entity, except that for many values that typically change in a benchmark, + * one can specify a sequence of values. + */ +@Value +@NoArgsConstructor +public class WorkloadSetTemplate { + public static final String DEFAULT_NAME_TEMPLATE = + "${topics}-topics-${partitionsPerTopic}-partitions-${messageSize}b" + + "-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}"; + public String nameFormat = DEFAULT_NAME_TEMPLATE; + + /** Number of topics to create in the test */ + public List topics = Collections.emptyList(); + /** Number of partitions each topic will contain */ + public List partitionsPerTopic= Collections.emptyList(); + public List messageSize= Collections.emptyList(); + public List subscriptionsPerTopic= Collections.emptyList(); + public List producersPerTopic= Collections.emptyList(); + public List consumerPerSubscription= Collections.emptyList(); + public List producerRate= Collections.emptyList(); + + public KeyDistributorType keyDistributor = KeyDistributorType.NO_KEY; + public String payloadFile = null; + public boolean useRandomizedPayloads = false; + public double randomBytesRatio = 0.0; + public int randomizedPayloadPoolSize = 0; + public long consumerBacklogSizeGB = 0; + public int testDurationMinutes = 5; + public int warmupDurationMinutes = 1; +} diff --git a/tool/src/test/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormatTest.java b/tool/src/test/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormatTest.java new file mode 100644 index 000000000..41a646f67 --- /dev/null +++ b/tool/src/test/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormatTest.java @@ -0,0 +1,46 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.tool.workload; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.openmessaging.benchmark.Workload; +import org.junit.jupiter.api.Test; + +class WorkloadNameFormatTest { + + public String nameFormat = "${topics}-topics-${partitionsPerTopic}-partitions-${messageSize}b" + + "-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}"; + + @Test + void nameOverride() { + Workload workload = new Workload(); + workload.name = "x"; + String name = new WorkloadNameFormat(nameFormat).from(workload); + assertThat(name).isEqualTo("x"); + } + + @Test + void from() { + Workload workload = new Workload(); + workload.topics = 1456; + workload.partitionsPerTopic = 2123; + workload.messageSize = 617890; + workload.producersPerTopic = 45; + workload.consumerPerSubscription = 541; + workload.producerRate = 1000000; + String name = new WorkloadNameFormat(nameFormat).from(workload); + assertThat(name).isEqualTo("1k-topics-2k-partitions-617kb-45p-541c-1m"); + } +} \ No newline at end of file diff --git a/tool/src/test/resources/template.yaml b/tool/src/test/resources/template.yaml new file mode 100644 index 000000000..e3895faec --- /dev/null +++ b/tool/src/test/resources/template.yaml @@ -0,0 +1,25 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +nameFormat: "${topics}-topics-${partitionsPerTopic}-partitions-${messageSize}b-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}" +topics: [1] +partitionsPerTopic: [1] +messageSize: [10000] +payloadFile: "payload/payload-100b.data" +subscriptionsPerTopic: [1] +consumerPerSubscription: [1, 2, 4, 8, 16, 32, 64] +producersPerTopic: [1, 2, 4, 8, 16, 32, 64] +producerRate: [50000] +consumerBacklogSizeGB: 0 +testDurationMinutes: 15 \ No newline at end of file From 3d04126e9cb1c5d4773676133d1caa38007aa44e Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Fri, 16 Sep 2022 17:05:29 +0100 Subject: [PATCH 044/103] Add backlog drain timings to the log (#303) * Add backlog drain timings to the log ## Motivation It can take time to determine how to scale up backlog drain workloads. How long it takes to build the backlog and drain it are useful pieces of information. ## Changes * Add timers to the log of the backlog build up time and backlog drain time * Add Timer.elapsedSeconds --- .../io/openmessaging/benchmark/WorkloadGenerator.java | 5 ++++- .../java/io/openmessaging/benchmark/utils/Timer.java | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index a8469808b..b737f02d8 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -334,6 +334,7 @@ private void createProducers(List topics) throws IOException { } private void buildAndDrainBacklog(List topics) throws IOException { + Timer timer = new Timer(); log.info("Stopping all consumers to build backlog"); worker.pauseConsumers(); @@ -357,6 +358,8 @@ private void buildAndDrainBacklog(List topics) throws IOException { } } + log.info("--- Completed backlog build in {} s ---", timer.elapsedSeconds()); + timer = new Timer(); log.info("--- Start draining backlog ---"); worker.resumeConsumers(); @@ -367,7 +370,7 @@ private void buildAndDrainBacklog(List topics) throws IOException { CountersStats stats = worker.getCountersStats(); long currentBacklog = workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived; if (currentBacklog <= minBacklog) { - log.info("--- Completed backlog draining ---"); + log.info("--- Completed backlog draining in {} s ---", timer.elapsedSeconds()); needToWaitForBacklogDraining = false; return; } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java index e71ffe330..da0e91aca 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java @@ -23,7 +23,15 @@ public Timer() { } public double elapsedMillis() { + return elapsed(TimeUnit.MILLISECONDS); + } + + public double elapsedSeconds() { + return elapsed(TimeUnit.SECONDS); + } + + private double elapsed(TimeUnit unit) { long now = System.nanoTime(); - return (now - startTime) / (double) TimeUnit.MILLISECONDS.toNanos(1); + return (now - startTime) / (double) unit.toNanos(1); } } From 146886c7ccf7b4f6122def8ebd2046d9370aacdc Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Fri, 16 Sep 2022 17:05:48 +0100 Subject: [PATCH 045/103] Move shutdown hook (#304) ## Motivation On normal completion, the shutdown hook still executes, but after the http client has been close resulting in exceptions in the log ## Changes * Move the shutdown hook into DistributedWorkersEnsemble * Remove the shutdown hook on close, before the http client is closed. --- .../src/main/java/io/openmessaging/benchmark/Benchmark.java | 2 -- .../benchmark/worker/DistributedWorkersEnsemble.java | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index 674ac3436..acb26eeba 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -14,7 +14,6 @@ package io.openmessaging.benchmark; import java.io.File; -import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; @@ -128,7 +127,6 @@ public static void main(String[] args) throws Exception { if (arguments.workers != null && !arguments.workers.isEmpty()) { worker = new DistributedWorkersEnsemble(arguments.workers, arguments.extraConsumers); - Runtime.getRuntime().addShutdownHook(new Thread(worker::stopAll)); } else { // Use local worker implementation worker = new LocalWorker(); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index cdf90b2f9..492656ee4 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -54,7 +54,7 @@ import io.openmessaging.benchmark.worker.commands.TopicsInfo; public class DistributedWorkersEnsemble implements Worker { - + private final Thread shutdownHook = new Thread(this::stopAll); private final List workers; private final List producerWorkers; private final List consumerWorkers; @@ -80,6 +80,8 @@ public DistributedWorkersEnsemble(List workers, boolean extraConsumerWor log.info("Workers list - producers: {}", producerWorkers); log.info("Workers list - consumers: {}", consumerWorkers); + + Runtime.getRuntime().addShutdownHook(shutdownHook); } @Override @@ -331,6 +333,7 @@ private CompletableFuture post(String host, String path, byte[] body, Cla @Override public void close() throws Exception { + Runtime.getRuntime().removeShutdownHook(shutdownHook); httpClient.close(); } From 5892855e236fbbd49774701fc88e7176db14b4fb Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 16 Sep 2022 09:18:53 -0700 Subject: [PATCH 046/103] Added license header to tool/pom.xml --- tool/pom.xml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tool/pom.xml b/tool/pom.xml index bd3f9ce24..cee260f0f 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -1,4 +1,19 @@ + @@ -57,4 +72,4 @@ - \ No newline at end of file + From 2b763f697f3da2111fdb87f77239364e72a3d344 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 16 Sep 2022 13:58:08 -0700 Subject: [PATCH 047/103] Fixed NATS driver to use JetStream API --- .../benchmark/worker/LocalWorker.java | 2 +- .../benchmark/driver/BenchmarkDriver.java | 2 +- driver-nats/nats.yaml | 20 +++ .../driver/nats/NatsBenchmarkConsumer.java | 7 +- .../driver/nats/NatsBenchmarkDriver.java | 120 ++++++++++++------ .../driver/nats/NatsBenchmarkProducer.java | 52 +++----- .../benchmark/driver/nats/NatsConfig.java | 2 + 7 files changed, 130 insertions(+), 75 deletions(-) create mode 100644 driver-nats/nats.yaml diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index bb1a95713..a2c9e4991 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -137,7 +137,7 @@ public void initializeDriver(File driverConfigFile) throws IOException { try { benchmarkDriver = (BenchmarkDriver) Class.forName(driverConfiguration.driverClass).newInstance(); benchmarkDriver.initialize(driverConfigFile, statsLogger); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException | InterruptedException e) { throw new RuntimeException(e); } } diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java index b4a0ba9a9..5a1f5b36d 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java @@ -34,7 +34,7 @@ public interface BenchmarkDriver extends AutoCloseable { * @param statsLogger stats logger to collect stats from benchmark driver * @throws IOException */ - void initialize(File configurationFile, StatsLogger statsLogger) throws IOException; + void initialize(File configurationFile, StatsLogger statsLogger) throws IOException, InterruptedException; /** * Get a driver specific prefix to be used in creating multiple topic names diff --git a/driver-nats/nats.yaml b/driver-nats/nats.yaml new file mode 100644 index 000000000..f5bbde94c --- /dev/null +++ b/driver-nats/nats.yaml @@ -0,0 +1,20 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: NATS +driverClass: io.openmessaging.benchmark.driver.nats.NatsBenchmarkDriver + +natsHostUrl: localhost:4222 + +replicationFactor: 1 diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java index 062dce260..8b5314bc6 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java @@ -17,11 +17,6 @@ import io.openmessaging.benchmark.driver.BenchmarkConsumer; public class NatsBenchmarkConsumer implements BenchmarkConsumer { - private Connection cn; - public NatsBenchmarkConsumer (Connection cn) { - this.cn = cn; - } - @Override public void close() throws Exception { - this.cn.close(); + @Override public void close() { } } diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index 8c40bf632..b6c2f9c70 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -17,8 +17,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.nats.client.Dispatcher; +import io.nats.client.ErrorListener; +import io.nats.client.JetStream; +import io.nats.client.JetStreamManagement; +import io.nats.client.JetStreamOptions; +import io.nats.client.JetStreamSubscription; +import io.nats.client.Message; +import io.nats.client.MessageHandler; import io.nats.client.Nats; import io.nats.client.Options; +import io.nats.client.PushSubscribeOptions; +import io.nats.client.api.StorageType; +import io.nats.client.api.StreamConfiguration; +import io.nats.client.api.StreamInfo; +import io.nats.client.support.JsonUtils; import java.time.Duration; import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.BenchmarkDriver; @@ -35,62 +47,98 @@ public class NatsBenchmarkDriver implements BenchmarkDriver { private NatsConfig config; - @Override public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException { + + private Connection connection; + private JetStream jetStream; + + @Override + public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException, InterruptedException { config = mapper.readValue(configurationFile, NatsConfig.class); log.info("read config file," + config.toString()); - } + this.connection = Nats.connect(new Options.Builder() + .server(config.natsHostUrl) + .maxReconnects(5) + .errorListener(new ErrorListener() { + @Override + public void errorOccurred(Connection conn, String error) { + log.error("Error on connection {}: {}", conn, error); + } - @Override public String getTopicNamePrefix() { - return "Nats-benchmark"; + @Override + public void exceptionOccurred(Connection conn, Exception exp) { + log.error("Exception on connection {}", conn, exp); + } + }) + .build()); + this.jetStream = connection.jetStream(); } - @Override public CompletableFuture createTopic(String topic, int partitions) { - log.info("nats create a topic" + topic); - log.info("ignore partitions"); - CompletableFuture future = new CompletableFuture<>(); - future.complete(null); - return future; + @Override + public String getTopicNamePrefix() { + return "Nats-benchmark"; } - @Override public CompletableFuture createProducer(String topic) { - Connection natsProducer; + @Override + public CompletableFuture createTopic(String topic, int partitions) { try { - Options options = new Options.Builder().server(config.natsHostUrl).maxReconnects(5).build(); - natsProducer = Nats.connect(options); + JetStreamManagement jsm = connection.jetStreamManagement(); + StreamInfo streamInfo = jsm.addStream(StreamConfiguration.builder() + .name(topic) + .subjects(topic) + .storageType(StorageType.File) + .replicas(config.replicationFactor) + .build()); + log.info("Created stream {} -- {}", topic, JsonUtils.getFormatted(streamInfo)); + return CompletableFuture.completedFuture(null); } catch (Exception e) { - log.error("createProducer excetopin " + e); - return null; + CompletableFuture f = new CompletableFuture<>(); + f.completeExceptionally(e); + return f; } - return CompletableFuture.completedFuture(new NatsBenchmarkProducer(natsProducer, topic)); } - @Override public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { - Dispatcher natsConsumer; - Connection cn; - log.info("createConsumer"); + @Override + public CompletableFuture createProducer(String topic) { + return CompletableFuture.completedFuture(new NatsBenchmarkProducer(jetStream, topic)); + } + + @Override + public CompletableFuture createConsumer(String topic, String subscriptionName, + ConsumerCallback consumerCallback) { + + + Dispatcher dispatcher = connection.createDispatcher(); + try { - Options options = new Options.Builder().server(config.natsHostUrl).maxReconnects(5).build(); - cn = Nats.connect(options); - natsConsumer = cn.createDispatcher((msg) -> { - consumerCallback.messageReceived(msg.getData(), Long.parseLong(msg.getReplyTo())); - }); - natsConsumer.subscribe(topic, subscriptionName); - cn.flush(Duration.ZERO); + JetStreamSubscription sub = jetStream.subscribe(topic, dispatcher, (Message msg) -> { + long publishTimestamp = readLongFromBytes(msg.getData()); + consumerCallback.messageReceived(msg.getData(), publishTimestamp); + msg.ack(); + }, false, new PushSubscribeOptions.Builder().build()); + return CompletableFuture.completedFuture(new NatsBenchmarkConsumer()); } catch (Exception e) { - log.error("createConsumer excetopin " + e); - return null; + CompletableFuture f = new CompletableFuture<>(); + f.completeExceptionally(e); + return f; } - log.info("createCOnsumer done"); - return CompletableFuture.completedFuture(new NatsBenchmarkConsumer(cn)); } - @Override public void close() throws Exception { - + @Override + public void close() throws Exception { + this.connection.close(); } private static final Logger log = LoggerFactory.getLogger(NatsBenchmarkDriver.class); private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + private static long readLongFromBytes(final byte[] b) { + long result = 0; + for (int i = 0; i < 8; i++) { + result <<= 8; + result |= (b[i] & 0xFF); + } + return result; + } } diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java index 1492404a5..71237c90f 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java @@ -13,7 +13,11 @@ */ package io.openmessaging.benchmark.driver.nats; +import io.nats.client.JetStream; +import io.nats.client.Message; +import io.nats.client.PublishOptions; import io.openmessaging.benchmark.driver.BenchmarkProducer; +import java.time.Duration; import java.util.Optional; import java.util.concurrent.CompletableFuture; import io.nats.client.Connection; @@ -25,44 +29,30 @@ public class NatsBenchmarkProducer implements BenchmarkProducer { private final String topic; - private final Connection natsProducer; - private final ExecutorService executor = Executors.newCachedThreadPool(); - private final Semaphore semaphore = new Semaphore(1000); - public NatsBenchmarkProducer(final Connection natsProducer, final String topic) { - this.natsProducer = natsProducer; + private final JetStream jetStream; + + public NatsBenchmarkProducer(JetStream jetStream, final String topic) { + this.jetStream = jetStream; this.topic = topic; } - @Override public CompletableFuture sendAsync(Optional key, byte[] payload) { - CompletableFuture future = new CompletableFuture<>(); - try { - semaphore.acquire(); + @Override + public CompletableFuture sendAsync(Optional key, byte[] payload) { + writeLongToBytes(System.currentTimeMillis(), payload); + return jetStream.publishAsync(topic, payload).thenApply(x -> null); + } - executor.submit(() -> { - try { - natsProducer.publish(topic, Long.toString(System.currentTimeMillis()), payload); -// natsProducer.flush(Duration.ofSeconds(1)); - } catch (Exception e) { - log.error("send exception" + e); - future.exceptionally(null); - } finally { - semaphore.release(); - } - future.complete(null); + @Override + public void close() throws Exception { + } - }); - } catch (Exception e) { - log.error("send exception", e); - future.exceptionally(null); - semaphore.release(); + public static void writeLongToBytes(long l, byte[] dst) { + for (int i = 7; i >= 0; i--) { + dst[i] = (byte) (l & 0xFF); + l >>= 8; } - return future; } - @Override public void close() throws Exception { - log.info("close a producer"); - natsProducer.close(); - } - private static final Logger log = LoggerFactory.getLogger(NatsBenchmarkProducer.class); + } diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java index ec2e40626..d16494f93 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java @@ -15,4 +15,6 @@ public class NatsConfig { public String natsHostUrl; + + public int replicationFactor; } From 7b9f069f678ae655ed3529be9e8f28c5aec326e4 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Tue, 20 Sep 2022 16:18:10 +0100 Subject: [PATCH 048/103] Cleanup logging (#305) ## Motivation log4j-slf4j-simpl and slf4j-simple are both on the classpath in various places which is messing up the logging. ## Changes * Replace slf4j-simple with log4j-slf4j-impl --- benchmark-framework/pom.xml | 4 +++ .../src/main/resources/log4j2.yaml | 4 +-- driver-api/pom.xml | 4 --- driver-kafka/pom.xml | 5 +++ driver-kafka/src/test/resources/log4j2.yaml | 33 +++++++++++++++++++ .../driver/kop/KopBenchmarkDriverTest.java | 2 +- pom.xml | 5 --- tool/pom.xml | 4 +-- .../src/main/resources/log4j2.yaml | 20 ++++++++--- 9 files changed, 63 insertions(+), 18 deletions(-) create mode 100644 driver-kafka/src/test/resources/log4j2.yaml rename driver-kafka/src/test/resources/simplelogger.properties => tool/src/main/resources/log4j2.yaml (62%) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index b09311aee..e21a892c5 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -113,6 +113,10 @@ prometheus-metrics-provider ${bookkeeper.version} + + org.apache.logging.log4j + log4j-slf4j-impl + io.javalin javalin diff --git a/benchmark-framework/src/main/resources/log4j2.yaml b/benchmark-framework/src/main/resources/log4j2.yaml index 4579aeff2..3083a28e3 100644 --- a/benchmark-framework/src/main/resources/log4j2.yaml +++ b/benchmark-framework/src/main/resources/log4j2.yaml @@ -21,13 +21,13 @@ Configuration: name: Console target: SYSTEM_OUT PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level - %msg%n" + Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level %c{1} - %msg%n" RollingFile: name: RollingFile fileName: benchmark-worker.log filePattern: benchmark-worker.log.%d{yyyy-MM-dd-hh-mm-ss}.gz PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level - %msg%n" + Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level %c{1} - %msg%n" Policies: SizeBasedTriggeringPolicy: size: 100MB diff --git a/driver-api/pom.xml b/driver-api/pom.xml index e5a635e2d..c26dd24d3 100644 --- a/driver-api/pom.xml +++ b/driver-api/pom.xml @@ -50,9 +50,5 @@ org.projectlombok lombok - - org.slf4j - slf4j-simple - diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index 0da9c2af0..58e1ebf69 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -42,6 +42,11 @@ org.assertj assertj-core + + org.apache.logging.log4j + log4j-slf4j-impl + test + org.junit.jupiter junit-jupiter diff --git a/driver-kafka/src/test/resources/log4j2.yaml b/driver-kafka/src/test/resources/log4j2.yaml new file mode 100644 index 000000000..bd1f7cfe1 --- /dev/null +++ b/driver-kafka/src/test/resources/log4j2.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Configuration: + status: INFO + name: driver-kafka-test + + Appenders: + Console: + name: Console + target: SYSTEM_OUT + PatternLayout: + Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level %c{1} - %msg%n" + Loggers: + Logger: + name: io.openmessaging.benchmark.driver.kafka + level: debug + Root: + level: info + additivity: false + AppenderRef: + - ref: Console diff --git a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java index 403515b7f..1b5533021 100644 --- a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java +++ b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java @@ -74,7 +74,7 @@ public void testLoadCustomConfig() throws Exception { assertFalse(pulsarConfig.blockIfQueueFull); assertEquals(pulsarConfig.pendingQueueSize, 10000); assertEquals(pulsarConfig.maxPendingMessagesAcrossPartitions, 500000); - assertEquals(pulsarConfig.batchingPartitionSwitchFrequencyByPublishDelay, 100); + assertEquals(pulsarConfig.batchingPartitionSwitchFrequencyByPublishDelay, 10); assertEquals(pulsarConfig.maxTotalReceiverQueueSizeAcrossPartitions, 500000); assertEquals(pulsarConfig.receiverQueueSize, 10000); diff --git a/pom.xml b/pom.xml index 2a12440c1..6b6689f3a 100644 --- a/pom.xml +++ b/pom.xml @@ -180,11 +180,6 @@ ${lombok.version} provided - - org.slf4j - slf4j-simple - ${slf4j.version} - diff --git a/tool/pom.xml b/tool/pom.xml index cee260f0f..73d344f36 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -41,8 +41,8 @@ jcommander - org.slf4j - slf4j-simple + org.apache.logging.log4j + log4j-slf4j-impl org.apache.commons diff --git a/driver-kafka/src/test/resources/simplelogger.properties b/tool/src/main/resources/log4j2.yaml similarity index 62% rename from driver-kafka/src/test/resources/simplelogger.properties rename to tool/src/main/resources/log4j2.yaml index a38a4f465..fa5945cb8 100644 --- a/driver-kafka/src/test/resources/simplelogger.properties +++ b/tool/src/main/resources/log4j2.yaml @@ -12,7 +12,19 @@ # limitations under the License. # -org.slf4j.simpleLogger.showDateTime=true -org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS -org.slf4j.simpleLogger.log.org.apache.kafka=WARN -org.slf4j.simpleLogger.log.io.openmessaging.benchmark.driver.kafka=DEBUG +Configuration: + status: INFO + name: driver-kafka-test + + Appenders: + Console: + name: Console + target: SYSTEM_OUT + PatternLayout: + Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level %c{1} - %msg%n" + Loggers: + Root: + level: info + additivity: false + AppenderRef: + - ref: Console From 76513dc8c09bf6351bed20f1f75d3d1e01b7d1bc Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 20 Sep 2022 16:19:01 +0100 Subject: [PATCH 049/103] Expose setters (#306) --- .../benchmark/tool/workload/WorkloadSetTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java index 32c588cf3..c94b48e1f 100644 --- a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java @@ -16,15 +16,15 @@ import io.openmessaging.benchmark.utils.distributor.KeyDistributorType; import java.util.Collections; import java.util.List; +import lombok.Data; import lombok.NoArgsConstructor; -import lombok.Value; /** * A template that defines a set of workload definitions. This is much like the * {@link io.openmessaging.benchmark.Workload} entity, except that for many values that typically change in a benchmark, * one can specify a sequence of values. */ -@Value +@Data @NoArgsConstructor public class WorkloadSetTemplate { public static final String DEFAULT_NAME_TEMPLATE = From 81cfdb977c059408700effb912a4a9b9c30ab138 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 20 Sep 2022 16:20:28 +0100 Subject: [PATCH 050/103] [CI] Introduce working CI (#307) * Update badges * Build workflow * Correct target branch * Add license information * Update messaging system list * License badge --- .github/workflows/pr-build-and-test.yml | 45 +++++++++++++++++++++++++ .travis.yml | 17 ---------- README.md | 19 ++++++----- 3 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/pr-build-and-test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/pr-build-and-test.yml b/.github/workflows/pr-build-and-test.yml new file mode 100644 index 000000000..b811b09f7 --- /dev/null +++ b/.github/workflows/pr-build-and-test.yml @@ -0,0 +1,45 @@ +name: Build and test + +on: + pull_request: + branches: + - master + push: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 17 + + - name: Build and Verify + run: mvn --no-transfer-progress --batch-mode verify + + - name: package surefire test results + if: failure() + run: | + rm -rf test-results + mkdir test-results + find . -type d -name "*surefire*" -exec cp --parents -R {} test-results/ \; + zip -r test-results.zip test-results + - uses: actions/upload-artifact@v3 + name: upload test-results + if: failure() + with: + name: test-results + path: test-results.zip diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f49b598ab..000000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -dist: trusty - -notifications: - email: - recipients: - - openmessaging@googlegroups.com - on_success: change - on_failure: always - -language: java - -jdk: - - oraclejdk8 - - oraclejdk11 - -script: - - mvn clean install diff --git a/README.md b/README.md index 8247c9fe2..a4ba2bc07 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,26 @@ # OpenMessaging Benchmark Framework -[![Build Status](https://app.travis-ci.com/openmessaging/benchmark.svg?branch=master)](https://app.travis-ci.com/openmessaging/benchmark) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/openmessaging/benchmark.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/openmessaging/benchmark/alerts/) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/openmessaging/benchmark.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/openmessaging/benchmark/context:python) -[![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/openmessaging/benchmark.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/openmessaging/benchmark/context:java) +[![Build](https://github.com/openmessaging/benchmark/actions/workflows/pr-build-and-test.yml/badge.svg)](https://github.com/openmessaging/benchmark/actions/workflows/pr-build-and-test.yml) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) - **Notice：** We do not consider or plan to release any unilateral test results based on this standard. For reference, you can purchase server tests on the cloud by yourself. This repository houses user-friendly, cloud-ready benchmarking suites for the following messaging platforms: +* [Apache ActiveMQ Artemis](https://activemq.apache.org/components/artemis/) +* [Apache Bookkeeper](https://bookkeeper.apache.org) * [Apache Kafka](https://kafka.apache.org) -* [Apache RocketMQ](https://rocketmq.apache.org) -* [RabbitMQ](https://www.rabbitmq.com/) * [Apache Pulsar](https://pulsar.apache.org) +* [Apache RocketMQ](https://rocketmq.apache.org) +* Generic [JMS](https://javaee.github.io/jms-spec/) +* [KoP (Kafka-on-Pulsar)](https://github.com/streamnative/kop) * [NATS Streaming](https://nats.io/) -* [Redis](https://redis.com/) +* [NSQ](https://nsq.io) * [Pravega](https://pravega.io/) +* [RabbitMQ](https://www.rabbitmq.com/) +* [Redis](https://redis.com/) > More details could be found at the [official documentation](http://openmessaging.cloud/docs/benchmarks/). +## License +Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 \ No newline at end of file From bf6f7e2393c571d018597cd3dad55e1662c688ce Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 20 Sep 2022 16:20:46 +0100 Subject: [PATCH 051/103] Remove unused (#308) --- benchmark-framework/pom.xml | 4 ---- pom.xml | 7 ------- 2 files changed, 11 deletions(-) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index e21a892c5..70c7cfff6 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -138,10 +138,6 @@ jetty-util ${jetty.version} - - junit - junit - diff --git a/pom.xml b/pom.xml index 6b6689f3a..cd5e887d8 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,6 @@ 1.18.24 2.13.2 1.48 - 4.13.1 4.1.65.Final 1.7.36 @@ -147,12 +146,6 @@ netty-all ${netty.version} - - junit - junit - ${junit.version} - test - org.assertj assertj-core From 6ec5ba80147e345332edba632add41feb126048c Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 20 Sep 2022 16:21:26 +0100 Subject: [PATCH 052/103] [CI] Include code quality plugins (#309) * Include code quality plugins * Plugin configs * Start at zero - this project has hardly any tests * Add organization Co-authored-by: Matteo Merli --- README.md | 21 +- dev/license-header.txt => etc/APACHE-2.txt | 0 etc/checkstyle.xml | 217 ++++++++++++++++++++ etc/findbugsExclude.xml | 19 ++ pom.xml | 223 ++++++++++++++++++++- 5 files changed, 472 insertions(+), 8 deletions(-) rename dev/license-header.txt => etc/APACHE-2.txt (100%) create mode 100644 etc/checkstyle.xml create mode 100644 etc/findbugsExclude.xml diff --git a/README.md b/README.md index a4ba2bc07..623bd9b16 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,25 @@ This repository houses user-friendly, cloud-ready benchmarking suites for the fo > More details could be found at the [official documentation](http://openmessaging.cloud/docs/benchmarks/). +## Build + +Requirements: + +* JDK 8 +* Maven 3.8.6+ + +Common build actions: + +| Action | Command | +|---------------------------------|------------------------------------------| +| Full build and test | `mvn clean verify` | +| Skip tests | `mvn clean verify -DskipTests` | +| Skip Jacoco test coverage check | `mvn clean verify -Djacoco.skip` | +| Skip Checkstyle standards check | `mvn clean verify -Dcheckstyle.skip` | +| Skip Spotless formatting check | `mvn clean verify -Dspotless.check.skip` | +| Format code | `mvn spotless:apply` | +| Generate license headers | `mvn license:format` | + ## License -Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 \ No newline at end of file +Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/dev/license-header.txt b/etc/APACHE-2.txt similarity index 100% rename from dev/license-header.txt rename to etc/APACHE-2.txt diff --git a/etc/checkstyle.xml b/etc/checkstyle.xml new file mode 100644 index 000000000..ffd04871c --- /dev/null +++ b/etc/checkstyle.xml @@ -0,0 +1,217 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/etc/findbugsExclude.xml b/etc/findbugsExclude.xml new file mode 100644 index 000000000..57e9cdfbc --- /dev/null +++ b/etc/findbugsExclude.xml @@ -0,0 +1,19 @@ + + + diff --git a/pom.xml b/pom.xml index cd5e887d8..33aa12481 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,11 @@ tool - + + OpenMessaging Project + https://openmessaging.cloud + + Apache License, Version 2.0 @@ -60,6 +64,7 @@ 3.23.1 4.14.4 + 10.3.3 3.12.0 2.17.1 1.18.24 @@ -67,17 +72,94 @@ 1.48 4.1.65.Final 1.7.36 + + 3.2.0 + 0.8.8 + 4.1 + 3.10.1 + 3.1.0 + 3.0.0-M7 + 4.7.2.0 + 2.25.0 + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.plugin.version} + + + + false + true + scope,groupId,artifactId + groupId,artifactId + groupId,artifactId + + + + + + + + + + **/*.md + + + + + + + etc/**/*.xml + .github/workflows/**/*.yml + **/doc/**/*.md + **/doc/**/*.puml + + + + + true + 4 + + + + + + + check + + check + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs.plugin.version} + + etc/findbugsExclude.xml + + + + + check + + + + + + com.mycila license-maven-plugin - 3.0 + ${license.plugin.version} -
dev/license-header.txt
- +
etc/APACHE-2.txt
LICENSE NOTICE @@ -92,19 +174,146 @@ SCRIPT_STYLE SCRIPT_STYLE XML_STYLE + SCRIPT_STYLE + APOSTROPHE_STYLE + + + com.mycila + license-maven-plugin-git + ${license.plugin.version} + + check + process-sources + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${checkstyle.plugin.version} + + true + true + true + etc/checkstyle.xml + true + + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + + + + check + + validate + + + + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven.enforcer.plugin.version} + + + enforce-maven-version + + enforce + + + + + [3.8.6,) + + + true + + + + + + maven-failsafe-plugin + ${maven.surefire.plugin.version} + + + + integration-test + verify + maven-surefire-plugin - 3.0.0-M7 + ${maven.surefire.plugin.version} + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.plugin.version} + + + + prepare-agent + + + + report + + report + + verify + + + check + + check + + verify + + + + BUNDLE + + + CLASS + COVEREDRATIO + 0.00 + + + METHOD + COVEREDRATIO + 0.00 + + + LINE + COVEREDRATIO + 0.00 + + + BRANCH + COVEREDRATIO + 0.00 + + + + + + + @@ -193,7 +402,7 @@ 8 8 8 - UTF-8 + ${project.build.sourceEncoding}} true true true @@ -218,7 +427,7 @@ 8 8 - UTF-8 + ${project.build.sourceEncoding} true true true From 8c4fabb976a5941a3950205a09cb37605560e664 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 20 Sep 2022 16:22:00 +0100 Subject: [PATCH 053/103] [RabbitMQ] Clean-up deploy (#310) * Use valid Ansible characters only * Tag for smaller redeploys * Separate tuned from mv * Name casing --- driver-rabbitmq/deploy/deploy.yaml | 16 ++++++++++++---- driver-rabbitmq/deploy/provision-rabbitmq-aws.tf | 4 ++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/driver-rabbitmq/deploy/deploy.yaml b/driver-rabbitmq/deploy/deploy.yaml index 27f35a595..ed339af14 100644 --- a/driver-rabbitmq/deploy/deploy.yaml +++ b/driver-rabbitmq/deploy/deploy.yaml @@ -90,6 +90,7 @@ state: restarted daemon_reload: yes name: "rabbitmq-server" + tags: [restart-rabbitmq] - name: RabbitMQ status shell: rabbitmqctl cluster_status @@ -145,7 +146,7 @@ daemon_reload: yes name: "chronyd" -- name: Rabbitmq benchmarking client setup +- name: RabbitMQ benchmarking client setup hosts: client connection: ssh become: true @@ -162,36 +163,43 @@ unarchive: src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz dest: /opt - - shell: > - tuned-adm profile latency-performance && - mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark + tags: [client-code] + - shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark + tags: [client-code] + - shell: tuned-adm profile latency-performance - template: src: "templates/workers.yaml" dest: "/opt/benchmark/workers.yaml" + tags: [client-code] - template: src: "templates/rabbitmq.yaml" dest: "/opt/benchmark/driver-rabbitmq/rabbitmq.yaml" + tags: [client-code] - name: Configure benchmark-worker memory lineinfile: dest: /opt/benchmark/bin/benchmark-worker regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB"' + tags: [client-code] - name: Configure benchmark memory lineinfile: dest: /opt/benchmark/bin/benchmark regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xmx4G"' + tags: [client-code] - name: Install benchmark-worker systemd service template: src: "templates/benchmark-worker.service" dest: "/etc/systemd/system/benchmark-worker.service" + tags: [client-code] - systemd: state: restarted daemon_reload: yes name: "benchmark-worker" + tags: [client-code] - name: List host addresses hosts: localhost diff --git a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf index d6bc0112a..e919f3c5a 100644 --- a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf +++ b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf @@ -105,7 +105,7 @@ resource "aws_instance" "rabbitmq" { count = "${var.num_instances["rabbitmq"]}" tags = { - Name = "rabbitmq-${count.index}" + Name = "rabbitmq_${count.index}" } } @@ -118,7 +118,7 @@ resource "aws_instance" "client" { count = "${var.num_instances["client"]}" tags = { - Name = "rabbitmq-client-${count.index}" + Name = "rabbitmq_client_${count.index}" } } From edd98689f089cbcbf8932a11d451cfb30f2f4adc Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 20 Sep 2022 17:32:25 +0100 Subject: [PATCH 054/103] [Bugfix] Temporarily skip checks (#311) * Add year for license, fix encoding * Skip checks until source regularization --- .github/workflows/pr-build-and-test.yml | 2 +- pom.xml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-build-and-test.yml b/.github/workflows/pr-build-and-test.yml index b811b09f7..7e662fa6f 100644 --- a/.github/workflows/pr-build-and-test.yml +++ b/.github/workflows/pr-build-and-test.yml @@ -28,7 +28,7 @@ jobs: java-version: 17 - name: Build and Verify - run: mvn --no-transfer-progress --batch-mode verify + run: mvn --no-transfer-progress --batch-mode verify -Dcheckstyle.skip -Dlicense.skip -Dspotless.check.skip -Dspotbugs.skip - name: package surefire test results if: failure() diff --git a/pom.xml b/pom.xml index 33aa12481..d46ec8695 100644 --- a/pom.xml +++ b/pom.xml @@ -49,7 +49,9 @@ OpenMessaging Project https://openmessaging.cloud - + + 2017 + Apache License, Version 2.0 @@ -402,7 +404,7 @@ 8 8 8 - ${project.build.sourceEncoding}} + ${project.build.sourceEncoding} true true true From e7ca4e76c6f1cb3daec0d9edbf5980d9805c9c3e Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 20 Sep 2022 17:46:48 +0100 Subject: [PATCH 055/103] [CI] Regularise licenses (#312) * Regularize headers * Enable license checks --- .github/workflows/pr-build-and-test.yml | 2 +- .../src/main/java/io/openmessaging/benchmark/Benchmark.java | 2 +- .../java/io/openmessaging/benchmark/DriverConfiguration.java | 2 +- .../main/java/io/openmessaging/benchmark/ResultsToCsv.java | 2 +- .../src/main/java/io/openmessaging/benchmark/TestResult.java | 2 +- .../src/main/java/io/openmessaging/benchmark/Workers.java | 2 +- .../src/main/java/io/openmessaging/benchmark/Workload.java | 2 +- .../java/io/openmessaging/benchmark/WorkloadGenerator.java | 2 +- .../java/io/openmessaging/benchmark/utils/ListPartition.java | 2 +- .../openmessaging/benchmark/utils/PaddingDecimalFormat.java | 2 +- .../io/openmessaging/benchmark/utils/RandomGenerator.java | 2 +- .../src/main/java/io/openmessaging/benchmark/utils/Timer.java | 2 +- .../io/openmessaging/benchmark/utils/UniformRateLimiter.java | 2 +- .../benchmark/utils/distributor/KeyDistributor.java | 2 +- .../benchmark/utils/distributor/KeyDistributorType.java | 2 +- .../benchmark/utils/distributor/KeyRoundRobin.java | 2 +- .../benchmark/utils/distributor/NoKeyDistributor.java | 2 +- .../openmessaging/benchmark/utils/distributor/RandomNano.java | 2 +- .../benchmark/utils/payload/FilePayloadReader.java | 2 +- .../benchmark/utils/payload/PayloadException.java | 2 +- .../openmessaging/benchmark/utils/payload/PayloadReader.java | 2 +- .../io/openmessaging/benchmark/worker/BenchmarkWorker.java | 2 +- .../benchmark/worker/DistributedWorkersEnsemble.java | 2 +- .../java/io/openmessaging/benchmark/worker/LocalWorker.java | 2 +- .../main/java/io/openmessaging/benchmark/worker/Worker.java | 2 +- .../java/io/openmessaging/benchmark/worker/WorkerHandler.java | 2 +- .../benchmark/worker/commands/ConsumerAssignment.java | 2 +- .../benchmark/worker/commands/CountersStats.java | 2 +- .../benchmark/worker/commands/CumulativeLatencies.java | 2 +- .../openmessaging/benchmark/worker/commands/PeriodStats.java | 2 +- .../benchmark/worker/commands/ProducerWorkAssignment.java | 2 +- .../benchmark/worker/commands/TopicSubscription.java | 2 +- .../openmessaging/benchmark/worker/commands/TopicsInfo.java | 2 +- .../io/openmessaging/benchmark/driver/BenchmarkConsumer.java | 2 +- .../io/openmessaging/benchmark/driver/BenchmarkDriver.java | 2 +- .../io/openmessaging/benchmark/driver/BenchmarkProducer.java | 2 +- .../io/openmessaging/benchmark/driver/ConsumerCallback.java | 2 +- .../benchmark/driver/artemis/ArtemisBenchmarkConsumer.java | 2 +- .../benchmark/driver/artemis/ArtemisBenchmarkDriver.java | 2 +- .../benchmark/driver/artemis/ArtemisBenchmarkProducer.java | 2 +- .../openmessaging/benchmark/driver/artemis/ArtemisConfig.java | 2 +- .../io/openmessaging/benchmark/driver/bookkeeper/Config.java | 2 +- .../benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java | 2 +- .../benchmark/driver/bookkeeper/DlogBenchmarkDriver.java | 2 +- .../benchmark/driver/bookkeeper/DlogBenchmarkProducer.java | 2 +- .../benchmark/driver/bookkeeper/stats/CounterAdaptor.java | 2 +- .../benchmark/driver/bookkeeper/stats/GaugeAdaptor.java | 2 +- .../driver/bookkeeper/stats/OpStatsLoggerAdaptor.java | 2 +- .../benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java | 2 +- .../benchmark/driver/jms/JMSBenchmarkConsumer.java | 2 +- .../benchmark/driver/jms/JMSBenchmarkDriver.java | 2 +- .../benchmark/driver/jms/JMSBenchmarkProducer.java | 2 +- .../benchmark/driver/jms/JMSBenchmarkTransactionProducer.java | 2 +- .../openmessaging/benchmark/driver/jms/config/JMSConfig.java | 2 +- .../java/io/openmessaging/benchmark/driver/kafka/Config.java | 2 +- .../benchmark/driver/kafka/KafkaBenchmarkConsumer.java | 2 +- .../benchmark/driver/kafka/KafkaBenchmarkDriver.java | 2 +- .../benchmark/driver/kafka/KafkaBenchmarkProducer.java | 2 +- .../benchmark/driver/kafka/KafkaTopicCreator.java | 2 +- .../benchmark/driver/kafka/KafkaTopicCreatorTest.java | 2 +- .../benchmark/driver/kop/KopBenchmarkDriver.java | 2 +- .../openmessaging/benchmark/driver/kop/config/ClientType.java | 2 +- .../io/openmessaging/benchmark/driver/kop/config/Config.java | 2 +- .../benchmark/driver/kop/config/PulsarConfig.java | 2 +- .../benchmark/driver/kop/KopBenchmarkDriverTest.java | 2 +- .../driver/natsStreaming/NatsStreamingBenchmarkConsumer.java | 2 +- .../driver/natsStreaming/NatsStreamingBenchmarkDriver.java | 2 +- .../driver/natsStreaming/NatsStreamingBenchmarkProducer.java | 2 +- .../driver/natsStreaming/NatsStreamingClientConfig.java | 2 +- .../benchmark/driver/nats/NatsBenchmarkConsumer.java | 2 +- .../benchmark/driver/nats/NatsBenchmarkDriver.java | 2 +- .../benchmark/driver/nats/NatsBenchmarkProducer.java | 2 +- .../io/openmessaging/benchmark/driver/nats/NatsConfig.java | 2 +- .../benchmark/driver/nsq/NsqBenchmarkConsumer.java | 2 +- .../benchmark/driver/nsq/NsqBenchmarkDriver.java | 2 +- .../benchmark/driver/nsq/NsqBenchmarkProducer.java | 2 +- .../java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java | 2 +- .../benchmark/driver/pravega/PravegaBenchmarkConsumer.java | 2 +- .../benchmark/driver/pravega/PravegaBenchmarkDriver.java | 2 +- .../benchmark/driver/pravega/PravegaBenchmarkProducer.java | 2 +- .../driver/pravega/PravegaBenchmarkTransactionProducer.java | 2 +- .../benchmark/driver/pravega/config/PravegaClientConfig.java | 2 +- .../benchmark/driver/pravega/config/PravegaConfig.java | 2 +- .../benchmark/driver/pravega/config/PravegaWriterConfig.java | 2 +- .../benchmark/driver/pulsar/PulsarBenchmarkConsumer.java | 2 +- .../benchmark/driver/pulsar/PulsarBenchmarkDriver.java | 2 +- .../benchmark/driver/pulsar/PulsarBenchmarkProducer.java | 2 +- .../benchmark/driver/pulsar/config/PulsarClientConfig.java | 2 +- .../benchmark/driver/pulsar/config/PulsarConfig.java | 2 +- .../benchmark/driver/pulsar/config/PulsarConsumerConfig.java | 2 +- .../benchmark/driver/pulsar/config/PulsarProducerConfig.java | 2 +- .../benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java | 2 +- .../benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java | 2 +- .../benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java | 2 +- .../benchmark/driver/rabbitmq/RabbitMqConfig.java | 2 +- .../benchmark/driver/redis/RedisBenchmarkConsumer.java | 2 +- .../benchmark/driver/redis/RedisBenchmarkDriver.java | 2 +- .../benchmark/driver/redis/RedisBenchmarkProducer.java | 2 +- .../benchmark/driver/redis/client/RedisClientConfig.java | 2 +- .../benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java | 2 +- .../benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java | 2 +- .../benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java | 2 +- .../driver/rocketmq/client/RocketMQClientConfig.java | 2 +- etc/checkstyle.xml | 4 +--- etc/findbugsExclude.xml | 4 +--- .../benchmark/tool/workload/WorkloadGenerationTool.java | 2 +- .../benchmark/tool/workload/WorkloadGenerator.java | 2 +- .../benchmark/tool/workload/WorkloadNameFormat.java | 2 +- .../benchmark/tool/workload/WorkloadSetTemplate.java | 2 +- .../benchmark/tool/workload/WorkloadNameFormatTest.java | 2 +- 110 files changed, 110 insertions(+), 114 deletions(-) diff --git a/.github/workflows/pr-build-and-test.yml b/.github/workflows/pr-build-and-test.yml index 7e662fa6f..7927b7fcc 100644 --- a/.github/workflows/pr-build-and-test.yml +++ b/.github/workflows/pr-build-and-test.yml @@ -28,7 +28,7 @@ jobs: java-version: 17 - name: Build and Verify - run: mvn --no-transfer-progress --batch-mode verify -Dcheckstyle.skip -Dlicense.skip -Dspotless.check.skip -Dspotbugs.skip + run: mvn --no-transfer-progress --batch-mode verify -Dcheckstyle.skip -Dspotless.check.skip -Dspotbugs.skip - name: package surefire test results if: failure() diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index acb26eeba..d698ef7d4 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java index 4597ab1a3..e220cfa2b 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java index 42910ea79..cb30a5646 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java index c13192391..62e4ae396 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java index 076b614c7..52da2c530 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java index a908f577f..759eb09af 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index b737f02d8..783420da1 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java index 745491d01..2af6cfca9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java index 7fc674228..1d9d91f18 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java index 8848b9a1f..2eba74b9e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java index da0e91aca..c7973eee3 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java index fdf99c6eb..829e118e9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java index 6ec3f2462..31453cb9a 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java index 5949a294f..e69d41d75 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java index afa8a1ff0..0a3182221 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java index 551875e1b..593f0e5d4 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/NoKeyDistributor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java index 20ecce0d0..f7265c140 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java index ea22c8bd9..27cd9463f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java index afb7e4988..a3887ac31 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadException.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java index 0318b55e9..97d82aa80 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/PayloadReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java index 47e351454..d6f9d52fa 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index 492656ee4..027f4f887 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index a2c9e4991..929dca337 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java index 57181b9d4..fa9dde4d9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java index a83bd0d8e..951fab4e9 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java index e7a90aa98..3ae9c97e7 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java index b28474838..0da132c47 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java index bb2484b8a..92a12ef7b 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java index 9f08be837..1d8c62b73 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java index 83f431199..53c7e8869 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java index d4eb57e40..f21ce6745 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java index 9b9fc4a0f..b277a8f5f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java index cc2c00385..1788dc6a5 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java index 5a1f5b36d..920e3c41e 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java index 1c2420177..da9bf06ac 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java index 8ae5f7254..88817009f 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java index 09a0ae833..c714e71fc 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java index 172b4f4f9..ae49824c1 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java index 81d8e483a..baaa8cf7c 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java index 701bc6638..418340b66 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java index 597080640..04b660c50 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java index 760c3e848..88657e6ee 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java index fff244d5e..b1ea6071c 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java index a35f8635a..ed2f667a3 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java index fdf294a61..61d359969 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java index ae3c38422..3900ef854 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java index 72c580d71..4f0c4fbdd 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java index f2627a20a..37337557d 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java index 157a2656b..ba2a0f520 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java index 12b715cb3..ef20d38d5 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java index d96533061..2a1718066 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java index 60d60c2d7..fc3ca70a6 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java index 0a87e401c..e550872d5 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java index 59e2a96dd..d2fafafa3 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/Config.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java index 9b8d1f7d5..171c852e5 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java index 6892e80dc..605d42257 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java index bcd80f5bc..a258173ba 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java index af1c949f5..d114d1521 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java b/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java index 034f2b3cb..ca5b084dc 100644 --- a/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java +++ b/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java index b87163455..d855d7c84 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java index f2e88a386..1e25a856e 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java index 1a8daeb8f..daff56a9c 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java index 595a504d9..d9c36bc09 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/PulsarConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java index 1b5533021..57f318963 100644 --- a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java +++ b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkConsumer.java b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkConsumer.java index c9952142d..72a35d40f 100644 --- a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkConsumer.java +++ b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkDriver.java b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkDriver.java index 2dd8430e8..2d1b079a1 100644 --- a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkDriver.java +++ b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkProducer.java b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkProducer.java index 2c4626ba3..52fa1e13a 100644 --- a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkProducer.java +++ b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingClientConfig.java b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingClientConfig.java index 84fb4fdce..57e45f426 100644 --- a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingClientConfig.java +++ b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingClientConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java index 8b5314bc6..fd44b4a2b 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index b6c2f9c70..3de6ea775 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java index 71237c90f..c9f856bb3 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java index d16494f93..f6770c982 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java index 4012a41e0..a301da2ca 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java index 617619242..65043b057 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java index 701d42950..1ae87645e 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java index c5e8189a7..5658105e2 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java index 20928b96c..d2b12b252 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java index 15cfd0e8f..2f8eefa53 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java index e59849408..fd9546fd3 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java index 883cf5100..8071caec3 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/PravegaBenchmarkTransactionProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java index be69af873..f1dd86b60 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaClientConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java index 30b7ac5e3..0853bb4b3 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java index 1466c7068..d470e8d31 100644 --- a/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java +++ b/driver-pravega/src/main/java/io/openmessaging/benchmark/driver/pravega/config/PravegaWriterConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java index 1d73ea9a6..f64db5f62 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java index ba4a06bf3..08778fc63 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java index cda35459f..cad7a4a12 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java index cbdebd39a..15e350b4e 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java index ec3512c63..8bcb24622 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java index 8d34d8819..981271eec 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java index 1528e4d74..0cfa555f4 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarProducerConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java index a7b30925a..975121e18 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index 370f7ae76..328bd1d0f 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java index 7aeaaeb3d..f795158ee 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java index 8fe2794cb..6518e44fe 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java index 73d41e4cf..c9d8b60a2 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java index 0e8b17215..9e5b28c87 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java index 71ca82802..d62c4c246 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java index 6bb220607..899f752ef 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/client/RedisClientConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java index 9e483ae71..a79b9ede1 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java index 6efefa2ec..8ac7c1f72 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java index 811e05d66..838808729 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java index a597c4057..840090452 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java @@ -1,4 +1,4 @@ -/** +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/etc/checkstyle.xml b/etc/checkstyle.xml index ffd04871c..c97ad7eb7 100644 --- a/etc/checkstyle.xml +++ b/etc/checkstyle.xml @@ -1,13 +1,11 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 545ded1c058681b99fcc851e7f5ea0710a8028ed Mon Sep 17 00:00:00 2001 From: Elliot West Date: Thu, 22 Sep 2022 17:02:40 +0100 Subject: [PATCH 057/103] [Bugfix] Do not allocate throughput to workers with no producers (#320) * Do not allocate throughput to workers with no producers * Logging --- .../benchmark/worker/DistributedWorkersEnsemble.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index 45caca017..64c22eae4 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -110,7 +110,8 @@ public void createProducers(List topics) { } // Number of actually used workers might be less than available workers - numberOfUsedProducerWorkers = i; + numberOfUsedProducerWorkers = (int) topicsPerProducerMap.values().stream().filter(t -> !t.isEmpty()).count(); + log.debug("Producing worker count: {} of {}", numberOfUsedProducerWorkers, producerWorkers.size()); CompletableFuture[] futures = topicsPerProducerMap.keySet().stream().map(producer -> { try { @@ -130,6 +131,7 @@ public void createProducers(List topics) { public void startLoad(ProducerWorkAssignment producerWorkAssignment) throws IOException { // Reduce the publish rate across all the brokers producerWorkAssignment.publishRate /= numberOfUsedProducerWorkers; + log.debug("Setting worker assigned publish rate to {} msgs/sec", producerWorkAssignment.publishRate); sendPost(producerWorkers, "/start-load", writer.writeValueAsBytes(producerWorkAssignment)); } @@ -142,6 +144,7 @@ public void probeProducers() throws IOException { public void adjustPublishRate(double publishRate) throws IOException { // Reduce the publish rate across all the brokers publishRate /= numberOfUsedProducerWorkers; + log.debug("Adjusting worker publish rate to {} msgs/sec", publishRate); sendPost(producerWorkers, "/adjust-publish-rate", writer.writeValueAsBytes(publishRate)); } From 5b3996971871997add7db4ffaabbf724da324c09 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Thu, 22 Sep 2022 17:04:10 +0100 Subject: [PATCH 058/103] [Feature] Collect producer error stats (#317) * Collect producer error stats * Add error rates to results * Create error chart * User return value * Prevent negative backlogs --- .../java/io/openmessaging/benchmark/Benchmark.java | 8 ++++++-- .../java/io/openmessaging/benchmark/TestResult.java | 1 + .../openmessaging/benchmark/WorkloadGenerator.java | 9 ++++++--- .../benchmark/worker/DistributedWorkersEnsemble.java | 3 +++ .../openmessaging/benchmark/worker/LocalWorker.java | 12 +++++++++++- .../benchmark/worker/commands/CountersStats.java | 1 + .../benchmark/worker/commands/PeriodStats.java | 2 ++ bin/create_charts.py | 4 ++++ 8 files changed, 34 insertions(+), 6 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index 5fb176e03..8d0728098 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -156,8 +156,12 @@ public static void main(String[] args) throws Exception { String fileName = useOutput? arguments.output: String.format("%s-%s-%s.json", workloadName, driverConfiguration.name, dateFormat.format(new Date())); - log.info("Writing test result into {}", fileName); - writer.writeValue(new File(fileName), result); + log.info("Writing test result into {}/{}", workloadName, fileName); + File folder = new File(workloadName); + if (!folder.mkdirs()) { + log.debug("Unable to create folder {}", folder); + } + writer.writeValue(new File(folder, fileName), result); generator.close(); } catch (Exception e) { diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java index 62e4ae396..88a11c0e2 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java @@ -28,6 +28,7 @@ public class TestResult { public int consumersPerTopic; public List publishRate = new ArrayList<>(); + public List publishErrorRate = new ArrayList<>(); public List consumeRate = new ArrayList<>(); public List backlog = new ArrayList<>(); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 783420da1..122a4efb3 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -414,16 +414,18 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw double publishRate = stats.messagesSent / elapsed; double publishThroughput = stats.bytesSent / elapsed / 1024 / 1024; + double errorRate = stats.messageSendErrors / elapsed; double consumeRate = stats.messagesReceived / elapsed; double consumeThroughput = stats.bytesReceived / elapsed / 1024 / 1024; - long currentBacklog = workload.subscriptionsPerTopic * stats.totalMessagesSent - - stats.totalMessagesReceived; + long currentBacklog = Math.max(0L, workload.subscriptionsPerTopic * stats.totalMessagesSent + - stats.totalMessagesReceived); log.info( - "Pub rate {} msg/s / {} MB/s | Cons rate {} msg/s / {} MB/s | Backlog: {} K | Pub Latency (ms) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {} | Pub Delay Latency (us) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {}", + "Pub rate {} msg/s / {} MB/s | Pub err {} err/s | Cons rate {} msg/s / {} MB/s | Backlog: {} K | Pub Latency (ms) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {} | Pub Delay Latency (us) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {}", rateFormat.format(publishRate), throughputFormat.format(publishThroughput), + rateFormat.format(errorRate), rateFormat.format(consumeRate), throughputFormat.format(consumeThroughput), dec.format(currentBacklog / 1000.0), // dec.format(microsToMillis(stats.publishLatency.getMean())), @@ -438,6 +440,7 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw throughputFormat.format(stats.publishDelayLatency.getMaxValue())); result.publishRate.add(publishRate); + result.publishErrorRate.add(errorRate); result.consumeRate.add(consumeRate); result.backlog.add(currentBacklog); result.publishLatencyAvg.add(microsToMillis(stats.publishLatency.getMean())); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index 64c22eae4..ad34ac808 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -196,10 +196,12 @@ public PeriodStats getPeriodStats() { PeriodStats stats = new PeriodStats(); individualStats.forEach(is -> { stats.messagesSent += is.messagesSent; + stats.messageSendErrors += is.messageSendErrors; stats.bytesSent += is.bytesSent; stats.messagesReceived += is.messagesReceived; stats.bytesReceived += is.bytesReceived; stats.totalMessagesSent += is.totalMessagesSent; + stats.totalMessageSendErrors += is.totalMessageSendErrors; stats.totalMessagesReceived += is.totalMessagesReceived; try { @@ -265,6 +267,7 @@ public CountersStats getCountersStats() throws IOException { individualStats.forEach(is -> { stats.messagesSent += is.messagesSent; stats.messagesReceived += is.messagesReceived; + stats.messageSendErrors += is.messageSendErrors; }); return stats; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index 929dca337..925dd635a 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -43,7 +43,6 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.RateLimiter; import io.netty.util.concurrent.DefaultThreadFactory; import io.openmessaging.benchmark.DriverConfiguration; @@ -77,7 +76,9 @@ public class LocalWorker implements Worker, ConsumerCallback { private final StatsLogger statsLogger; private final LongAdder messagesSent = new LongAdder(); + private final LongAdder messageSendErrors = new LongAdder(); private final LongAdder bytesSent = new LongAdder(); + private final Counter messageSendErrorCounter; private final Counter messagesSentCounter; private final Counter bytesSentCounter; @@ -87,6 +88,7 @@ public class LocalWorker implements Worker, ConsumerCallback { private final Counter bytesReceivedCounter; private final LongAdder totalMessagesSent = new LongAdder(); + private final LongAdder totalMessageSendErrors = new LongAdder(); private final LongAdder totalMessagesReceived = new LongAdder(); private final static long highestTrackableValue = TimeUnit.SECONDS.toMicros(60); @@ -115,6 +117,7 @@ public LocalWorker(StatsLogger statsLogger) { StatsLogger producerStatsLogger = statsLogger.scope("producer"); this.messagesSentCounter = producerStatsLogger.getCounter("messages_sent"); + this.messageSendErrorCounter = producerStatsLogger.getCounter("message_send_errors"); this.bytesSentCounter = producerStatsLogger.getCounter("bytes_sent"); this.publishDelayLatencyStats = producerStatsLogger.getOpStatsLogger("producer_delay_latency"); this.publishLatencyStats = producerStatsLogger.getOpStatsLogger("produce_latency"); @@ -242,6 +245,9 @@ private void submitProducersToExecutor(List producers, KeyDis cumulativePublishDelayLatencyRecorder.recordValue(sendDelayMicros); publishDelayLatencyStats.registerSuccessfulEvent(sendDelayMicros, TimeUnit.MICROSECONDS); }).exceptionally(ex -> { + messageSendErrors.increment(); + messageSendErrorCounter.inc(); + totalMessageSendErrors.increment(); log.warn("Write error on message", ex); return null; }); @@ -267,12 +273,14 @@ public PeriodStats getPeriodStats() { PeriodStats stats = new PeriodStats(); stats.messagesSent = messagesSent.sumThenReset(); + stats.messageSendErrors = messageSendErrors.sumThenReset(); stats.bytesSent = bytesSent.sumThenReset(); stats.messagesReceived = messagesReceived.sumThenReset(); stats.bytesReceived = bytesReceived.sumThenReset(); stats.totalMessagesSent = totalMessagesSent.sum(); + stats.totalMessageSendErrors = totalMessageSendErrors.sum(); stats.totalMessagesReceived = totalMessagesReceived.sum(); stats.publishLatency = publishLatencyRecorder.getIntervalHistogram(); @@ -294,6 +302,7 @@ public CumulativeLatencies getCumulativeLatencies() { public CountersStats getCountersStats() throws IOException { CountersStats stats = new CountersStats(); stats.messagesSent = totalMessagesSent.sum(); + stats.messageSendErrors = totalMessageSendErrors.sum(); stats.messagesReceived = totalMessagesReceived.sum(); return stats; } @@ -367,6 +376,7 @@ public void stopAll() { endToEndCumulativeLatencyRecorder.reset(); messagesSent.reset(); + messageSendErrors.reset(); bytesSent.reset(); messagesReceived.reset(); bytesReceived.reset(); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java index 0da132c47..a7f935f86 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java @@ -16,4 +16,5 @@ public class CountersStats { public long messagesSent; public long messagesReceived; + public long messageSendErrors; } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java index 1d8c62b73..2dbe57d72 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java @@ -21,12 +21,14 @@ public class PeriodStats { public long messagesSent = 0; + public long messageSendErrors = 0; public long bytesSent = 0; public long messagesReceived = 0; public long bytesReceived = 0; public long totalMessagesSent = 0; + public long totalMessageSendErrors = 0; public long totalMessagesReceived = 0; @JsonIgnore diff --git a/bin/create_charts.py b/bin/create_charts.py index d326edd03..909051620 100755 --- a/bin/create_charts.py +++ b/bin/create_charts.py @@ -46,6 +46,10 @@ def create_charts(test_results): y_label='Rate (msg/s)', time_series=[(x['driver'], x['publishRate']) for x in results]) + create_chart(workload, 'Publish Error rate', + y_label='Rate (err/s)', + time_series=[(x['driver'], x['publishErrorRate']) for x in results]) + create_chart(workload, 'End To End Latency 95pct', y_label='Latency (ms)', time_series=[(x['driver'], x['endToEndLatency95pct']) for x in results]) From 519ae6859303cf1edb7e9a835605a88ea719ab1d Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 22 Sep 2022 17:04:27 +0100 Subject: [PATCH 059/103] Fix spotless with markdown (#318) Partially addresses #313 ## Motivation The current spotless configuration is applies twice with slightly differing results so a `spotless:apply` then fails a `spotless:check` ## Changes * Remove md files from the general formats so that only markdown flexmark is applied --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index d46ec8695..a52d7f98d 100644 --- a/pom.xml +++ b/pom.xml @@ -118,7 +118,6 @@ etc/**/*.xml .github/workflows/**/*.yml - **/doc/**/*.md **/doc/**/*.puml From c6f77f30c54da6c268d574fd1c9cb866bb23eec5 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 22 Sep 2022 17:04:39 +0100 Subject: [PATCH 060/103] Allow checkstyle to be suppressed in specific cases (#319) Partially addresses #313 ## Motivation Sometimes it makes sense to be able to ignore checkstyle warnings ## Changes * Enable the SuppressWarnings checkstyle configuration --- etc/checkstyle.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/checkstyle.xml b/etc/checkstyle.xml index c97ad7eb7..ce3711e00 100644 --- a/etc/checkstyle.xml +++ b/etc/checkstyle.xml @@ -109,10 +109,10 @@ - + - + From f093dc30b22af39270d6620b5c1b5e2bffa99d48 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 22 Sep 2022 19:13:59 +0100 Subject: [PATCH 061/103] Allow spaces in empty for iterator (#321) Partially addresses #313 ## Motivation spotless (googleJavaFormat) expects whitespace in an empty for iterator, but checkstyle does not. ## Changes * Update checkstyle configuration to allow whitespace --- etc/checkstyle.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/checkstyle.xml b/etc/checkstyle.xml index ce3711e00..2d95c2bf3 100644 --- a/etc/checkstyle.xml +++ b/etc/checkstyle.xml @@ -151,7 +151,9 @@ - + + + From 5e8f2aec02f64437d6b75390a6a96600ed8c6bcf Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 27 Sep 2022 16:36:27 +0100 Subject: [PATCH 062/103] [RabbitMQ] Expose management port (#323) * Expose management port * Document management interface --- driver-rabbitmq/README.md | 11 ++++++++++- driver-rabbitmq/deploy/provision-rabbitmq-aws.tf | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/driver-rabbitmq/README.md b/driver-rabbitmq/README.md index daca66070..16ef49782 100644 --- a/driver-rabbitmq/README.md +++ b/driver-rabbitmq/README.md @@ -111,4 +111,13 @@ You can also run specific workloads in the `workloads` folder. Here's an example ```bash $ sudo bin/benchmark --drivers driver-rabbotmq/rabbitmq.yaml workloads/1-topic-1-partitions-1kb.yaml -``` \ No newline at end of file +``` + +## Monitoring + +The `rabbitmq_management` plugin is installed, and the HTTP endpoint is exposed on all RabbitMQ instances on port +`15672`. This allows access to the management UI which provides some basic status metrics. It should also be possible +to access the management REST API at this endpoint. + +Note that the connection is authenticated but not currently encrypted and so passwords will be passed in plain text. Use +the `admin` account configured in the [Terraform](deploy/provision-rabbitmq-aws.tf) file to log in. \ No newline at end of file diff --git a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf index e919f3c5a..72f4fe746 100644 --- a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf +++ b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf @@ -70,6 +70,14 @@ resource "aws_security_group" "benchmark_security_group" { cidr_blocks = ["0.0.0.0/0"] } + # Management UI access from anywhere + ingress { + from_port = 15672 + to_port = 15672 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + # All ports open within the VPC ingress { from_port = 0 From a1b5bc377bc1dc353e6bda215baaf75c105c0e31 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 27 Sep 2022 16:37:38 +0100 Subject: [PATCH 063/103] Use quorum queues (#325) classic replicated queues have been deprecated --- .../benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index 328bd1d0f..6181d15c5 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -123,7 +123,7 @@ public CompletableFuture createConsumer(String topic, String String queueName = exchange + "-" + subscriptionName; channel.exchangeDeclare(exchange, BuiltinExchangeType.FANOUT, true); // Create the queue - channel.queueDeclare(queueName, true, false, false, Collections.emptyMap()); + channel.queueDeclare(queueName, true, false, false, Collections.singletonMap("x-queue-type", "quorum")); channel.queueBind(queueName, exchange, ""); future.complete(new RabbitMqBenchmarkConsumer(channel, queueName, consumerCallback)); } catch (IOException e) { From 929c4c302521ec9df59b5164966d86c6d34fa0a2 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Wed, 28 Sep 2022 18:22:15 +0100 Subject: [PATCH 064/103] [Unit Tests] Basic tests 1 (#326) * Extract test deps * Rate limiter test * Timer test * Decimal format test * List partition test * Licenses --- benchmark-framework/pom.xml | 12 ++++ .../openmessaging/benchmark/utils/Timer.java | 11 +++- .../benchmark/utils/UniformRateLimiter.java | 12 +++- .../benchmark/utils/ListPartitionTest.java | 34 ++++++++++ .../utils/PaddingDecimalFormatTest.java | 29 +++++++++ .../benchmark/utils/TimerTest.java | 41 ++++++++++++ .../utils/UniformRateLimiterTest.java | 63 +++++++++++++++++++ driver-kafka/pom.xml | 4 -- pom.xml | 14 +++++ 9 files changed, 212 insertions(+), 8 deletions(-) create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index 70c7cfff6..997a5ab45 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -138,6 +138,18 @@ jetty-util ${jetty.version} + + org.assertj + assertj-core + + + org.junit.jupiter + junit-jupiter + + + org.mockito + mockito-junit-jupiter + diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java index c7973eee3..04144d679 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java @@ -14,12 +14,19 @@ package io.openmessaging.benchmark.utils; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; public class Timer { private final long startTime; + private final Supplier nanoClock; + + Timer(Supplier nanoClock) { + this.nanoClock = nanoClock; + startTime = this.nanoClock.get(); + } public Timer() { - startTime = System.nanoTime(); + this(() -> System.nanoTime()); } public double elapsedMillis() { @@ -31,7 +38,7 @@ public double elapsedSeconds() { } private double elapsed(TimeUnit unit) { - long now = System.nanoTime(); + long now = nanoClock.get(); return (now - startTime) / (double) unit.toNanos(1); } } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java index 829e118e9..187cbf9c2 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java @@ -13,9 +13,11 @@ */ package io.openmessaging.benchmark.utils; +import java.time.Clock; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.locks.LockSupport; +import java.util.function.Supplier; /** * Provides a next operation time for rate limited operation streams.
@@ -32,8 +34,9 @@ public final class UniformRateLimiter { private volatile long virtualTime; private final double opsPerSec; private final long intervalNs; + private final Supplier nanoClock; - public UniformRateLimiter(final double opsPerSec) { + UniformRateLimiter(final double opsPerSec, Supplier nanoClock) { if (Double.isNaN(opsPerSec) || Double.isInfinite(opsPerSec)) { throw new IllegalArgumentException("opsPerSec cannot be Nan or Infinite"); } @@ -42,6 +45,11 @@ public UniformRateLimiter(final double opsPerSec) { } this.opsPerSec = opsPerSec; intervalNs = Math.round(ONE_SEC_IN_NS / opsPerSec); + this.nanoClock = nanoClock; + + } + public UniformRateLimiter(final double opsPerSec) { + this(opsPerSec, () -> System.nanoTime()); } public double getOpsPerSec() { @@ -56,7 +64,7 @@ public long acquire() { final long currOpIndex = V_TIME_UPDATER.getAndIncrement(this); long start = this.start; if (start == Long.MIN_VALUE) { - start = System.nanoTime(); + start = nanoClock.get(); if (!START_UPDATER.compareAndSet(this, Long.MIN_VALUE, start)) { start = this.start; assert start != Long.MIN_VALUE; diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java new file mode 100644 index 000000000..b3e7c0a53 --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.utils; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import java.util.List; +import org.junit.jupiter.api.Test; + +class ListPartitionTest { + + @Test + void partitionList() { + List list = asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + List> lists = ListPartition.partitionList(list, 3); + assertThat(lists).satisfies(s -> { + assertThat(s).hasSize(3); + assertThat(s.get(0)).isEqualTo(asList(1, 4, 7, 10)); + assertThat(s.get(1)).isEqualTo(asList(2, 5, 8)); + assertThat(s.get(2)).isEqualTo(asList(3, 6, 9)); + }); + } +} \ No newline at end of file diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java new file mode 100644 index 000000000..dfe52031a --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.utils; + +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.jupiter.api.Test; + +class PaddingDecimalFormatTest { + + @Test + void format() { + PaddingDecimalFormat format = new PaddingDecimalFormat("0.0", 7); + assertThat(format.format(1L)).isEqualTo(" 1.0"); + assertThat(format.format(1000L)).isEqualTo(" 1000.0"); + assertThat(format.format(10000000L)).isEqualTo("10000000.0"); + } + +} \ No newline at end of file diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java new file mode 100644 index 000000000..5d52fbe01 --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.utils; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; + +class TimerTest { + + @Test + void elapsedMillis() { + Supplier mockClock = mock(Supplier.class); + when(mockClock.get()).thenReturn(MILLISECONDS.toNanos(1), MILLISECONDS.toNanos(3)); + Timer timer = new Timer(mockClock); + assertThat(timer.elapsedMillis()).isEqualTo(2.0d); + } + + @Test + void elapsedSeconds() { + Supplier mockClock = mock(Supplier.class); + when(mockClock.get()).thenReturn(SECONDS.toNanos(1), SECONDS.toNanos(3)); + Timer timer = new Timer(mockClock); + assertThat(timer.elapsedSeconds()).isEqualTo(2.0d); + } +} \ No newline at end of file diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java new file mode 100644 index 000000000..b9f200846 --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.utils; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; + +class UniformRateLimiterTest { + + @Test + void getOpsPerSec() { + assertThat(new UniformRateLimiter(1000).getOpsPerSec()).isEqualTo(1000.0d); + } + + @Test + void getIntervalNs() { + assertThat(new UniformRateLimiter(1000).getIntervalNs()).isEqualTo(SECONDS.toNanos(1) / 1000); + } + + @Test + void acquireSlowSingleThread() { + Supplier mockClock = mock(Supplier.class); + when(mockClock.get()).thenReturn(SECONDS.toNanos(2)); + UniformRateLimiter rateLimiter = new UniformRateLimiter(1000, mockClock); + assertThat(rateLimiter.acquire()).isEqualTo(2000000000L); + assertThat(rateLimiter.acquire()).isEqualTo(2001000000L); + assertThat(rateLimiter.acquire()).isEqualTo(2002000000L); + } + + @Test + void uninterruptibleSleepNs() { + long start = System.nanoTime(); + long expectedEnd = start + MILLISECONDS.toNanos(100); + UniformRateLimiter.uninterruptibleSleepNs(expectedEnd); + long end = System.nanoTime(); + assertThat(end).isGreaterThan(expectedEnd); + } + + @Test + void cinitExceptions() { + assertThatCode(() -> new UniformRateLimiter(Double.NaN)).isInstanceOf(IllegalArgumentException.class); + assertThatCode(() -> new UniformRateLimiter(1.0d / 0.0d)).isInstanceOf(IllegalArgumentException.class); + assertThatCode(() -> new UniformRateLimiter(-0.1)).isInstanceOf(IllegalArgumentException.class); + assertThatCode(() -> new UniformRateLimiter(0.0)).isInstanceOf(IllegalArgumentException.class); + } +} \ No newline at end of file diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index 58e1ebf69..6619a4cc9 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -50,14 +50,10 @@ org.junit.jupiter junit-jupiter - 5.9.0 - test org.mockito mockito-junit-jupiter - 4.7.0 - test diff --git a/pom.xml b/pom.xml index a52d7f98d..f36d00ab4 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,8 @@ 1.18.24 2.13.2 1.48 + 5.9.0 + 4.8.0 4.1.65.Final 1.7.36 @@ -383,6 +385,18 @@ ${lombok.version} provided + + org.junit.jupiter + junit-jupiter + ${junit.jupiter.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.junit.jupiter.version} + test + From d5141e1d2ba234f2bab8426cfe9419d53bca0cce Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Thu, 29 Sep 2022 16:40:39 +0100 Subject: [PATCH 065/103] Reformat code (#324) Fixes #313 ## Motivation Apply consistent format to source code and enable checks in the build. ## Changes * Run mvn spotless:apply * Manually fixed or suppressed checkstyle warnings * reinstate 4 spaces indent --- .github/workflows/pr-build-and-test.yml | 2 +- README.md | 5 +- benchmark-framework/pom.xml | 311 ++++--- .../io/openmessaging/benchmark/Benchmark.java | 156 ++-- .../benchmark/DriverConfiguration.java | 1 - .../openmessaging/benchmark/ResultsToCsv.java | 82 +- .../openmessaging/benchmark/TestResult.java | 1 + .../io/openmessaging/benchmark/Workers.java | 1 + .../io/openmessaging/benchmark/Workload.java | 13 +- .../benchmark/WorkloadGenerator.java | 281 +++--- .../benchmark/utils/ListPartition.java | 6 +- .../benchmark/utils/PaddingDecimalFormat.java | 12 +- .../benchmark/utils/RandomGenerator.java | 1 + .../openmessaging/benchmark/utils/Timer.java | 1 + .../benchmark/utils/UniformRateLimiter.java | 4 +- .../utils/distributor/KeyDistributor.java | 5 +- .../utils/distributor/KeyDistributorType.java | 13 +- .../utils/distributor/KeyRoundRobin.java | 1 + .../utils/distributor/RandomNano.java | 1 + .../utils/payload/FilePayloadReader.java | 10 +- .../benchmark/worker/BenchmarkWorker.java | 37 +- .../worker/DistributedWorkersEnsemble.java | 373 ++++---- .../benchmark/worker/LocalWorker.java | 225 +++-- .../benchmark/worker/Worker.java | 6 +- .../benchmark/worker/WorkerHandler.java | 29 +- .../worker/commands/ConsumerAssignment.java | 1 + .../worker/commands/CumulativeLatencies.java | 12 +- .../worker/commands/PeriodStats.java | 13 +- .../commands/ProducerWorkAssignment.java | 6 +- .../worker/commands/TopicSubscription.java | 15 +- .../benchmark/worker/commands/TopicsInfo.java | 3 +- .../benchmark/utils/ListPartitionTest.java | 17 +- .../utils/PaddingDecimalFormatTest.java | 4 +- .../benchmark/utils/TimerTest.java | 3 +- .../utils/UniformRateLimiterTest.java | 9 +- deployment/kubernetes/helm/README.md | 3 +- docker/README.md | 10 +- docker/pom.xml | 23 +- driver-api/pom.xml | 71 +- .../benchmark/driver/BenchmarkConsumer.java | 4 +- .../benchmark/driver/BenchmarkDriver.java | 59 +- .../benchmark/driver/BenchmarkProducer.java | 8 +- .../benchmark/driver/ConsumerCallback.java | 23 +- driver-artemis/pom.xml | 42 +- .../artemis/ArtemisBenchmarkConsumer.java | 42 +- .../artemis/ArtemisBenchmarkDriver.java | 76 +- .../artemis/ArtemisBenchmarkProducer.java | 19 +- driver-bookkeeper/pom.xml | 49 +- .../benchmark/driver/bookkeeper/Config.java | 1 - .../bookkeeper/DlogBenchmarkConsumer.java | 106 ++- .../bookkeeper/DlogBenchmarkDriver.java | 97 +- .../bookkeeper/DlogBenchmarkProducer.java | 9 +- .../bookkeeper/stats/CounterAdaptor.java | 1 + .../driver/bookkeeper/stats/GaugeAdaptor.java | 1 + .../stats/OpStatsLoggerAdaptor.java | 1 + .../bookkeeper/stats/StatsLoggerAdaptor.java | 13 +- driver-jms/README.md | 9 +- driver-jms/pom.xml | 55 +- .../driver/jms/JMSBenchmarkConsumer.java | 45 +- .../driver/jms/JMSBenchmarkDriver.java | 92 +- .../driver/jms/JMSBenchmarkProducer.java | 54 +- .../jms/JMSBenchmarkTransactionProducer.java | 88 +- .../driver/jms/config/JMSConfig.java | 4 +- driver-kafka/deploy/README.md | 6 +- driver-kafka/pom.xml | 87 +- .../driver/kafka/KafkaBenchmarkConsumer.java | 71 +- .../driver/kafka/KafkaBenchmarkDriver.java | 46 +- .../driver/kafka/KafkaBenchmarkProducer.java | 23 +- .../driver/kafka/KafkaTopicCreator.java | 39 +- .../driver/kafka/KafkaTopicCreatorTest.java | 12 +- driver-kop/pom.xml | 79 +- .../driver/kop/KopBenchmarkDriver.java | 111 ++- .../driver/kop/config/ClientType.java | 7 +- .../benchmark/driver/kop/config/Config.java | 1 + .../driver/kop/KopBenchmarkDriverTest.java | 6 +- driver-nats-streaming/pom.xml | 21 +- .../NatsStreamingBenchmarkConsumer.java | 6 +- .../NatsStreamingBenchmarkDriver.java | 96 +- .../NatsStreamingBenchmarkProducer.java | 33 +- driver-nats/pom.xml | 20 +- .../driver/nats/NatsBenchmarkConsumer.java | 6 +- .../driver/nats/NatsBenchmarkDriver.java | 84 +- .../driver/nats/NatsBenchmarkProducer.java | 15 +- driver-nsq/pom.xml | 20 +- .../driver/nsq/NsqBenchmarkConsumer.java | 6 +- .../driver/nsq/NsqBenchmarkDriver.java | 51 +- .../driver/nsq/NsqBenchmarkProducer.java | 38 +- driver-pravega/README.md | 72 +- driver-pravega/doc/build_pravega.md | 5 + driver-pravega/doc/metrics_and_logs.md | 14 +- driver-pravega/doc/run_in_k8s.md | 3 + driver-pravega/doc/troubleshooting.md | 1 + driver-pravega/pom.xml | 49 +- .../pravega/PravegaBenchmarkConsumer.java | 79 +- .../pravega/PravegaBenchmarkDriver.java | 69 +- .../pravega/PravegaBenchmarkProducer.java | 28 +- .../PravegaBenchmarkTransactionProducer.java | 42 +- .../driver/pravega/config/PravegaConfig.java | 3 +- driver-pulsar/README.md | 1 + driver-pulsar/pom.xml | 55 +- .../pulsar/PulsarBenchmarkConsumer.java | 2 +- .../driver/pulsar/PulsarBenchmarkDriver.java | 191 ++-- .../pulsar/PulsarBenchmarkProducer.java | 6 +- .../pulsar/config/PulsarClientConfig.java | 1 + driver-rabbitmq/README.md | 27 +- driver-rabbitmq/pom.xml | 49 +- .../rabbitmq/RabbitMqBenchmarkConsumer.java | 12 +- .../rabbitmq/RabbitMqBenchmarkDriver.java | 134 +-- .../rabbitmq/RabbitMqBenchmarkProducer.java | 103 ++- .../driver/rabbitmq/RabbitMqConfig.java | 1 + driver-redis/pom.xml | 16 +- .../driver/redis/RedisBenchmarkConsumer.java | 74 +- .../driver/redis/RedisBenchmarkDriver.java | 57 +- .../driver/redis/RedisBenchmarkProducer.java | 19 +- driver-rocketmq/pom.xml | 16 +- .../rocketmq/RocketMQBenchmarkConsumer.java | 1 + .../rocketmq/RocketMQBenchmarkDriver.java | 89 +- .../rocketmq/RocketMQBenchmarkProducer.java | 28 +- package/pom.xml | 87 +- pom.xml | 864 +++++++++--------- tool/README.md | 9 +- tool/pom.xml | 107 ++- .../tool/workload/WorkloadGenerationTool.java | 27 +- .../tool/workload/WorkloadGenerator.java | 1 + .../tool/workload/WorkloadNameFormat.java | 15 +- .../tool/workload/WorkloadSetTemplate.java | 26 +- .../tool/workload/WorkloadNameFormatTest.java | 7 +- 127 files changed, 3279 insertions(+), 2660 deletions(-) diff --git a/.github/workflows/pr-build-and-test.yml b/.github/workflows/pr-build-and-test.yml index 659c812e4..b811b09f7 100644 --- a/.github/workflows/pr-build-and-test.yml +++ b/.github/workflows/pr-build-and-test.yml @@ -28,7 +28,7 @@ jobs: java-version: 17 - name: Build and Verify - run: mvn --no-transfer-progress --batch-mode verify -Dcheckstyle.skip -Dspotless.check.skip + run: mvn --no-transfer-progress --batch-mode verify - name: package surefire test results if: failure() diff --git a/README.md b/README.md index 623bd9b16..9a1c83634 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # OpenMessaging Benchmark Framework + [![Build](https://github.com/openmessaging/benchmark/actions/workflows/pr-build-and-test.yml/badge.svg)](https://github.com/openmessaging/benchmark/actions/workflows/pr-build-and-test.yml) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -**Notice：** We do not consider or plan to release any unilateral test results based on this standard. For reference, you can purchase server tests on the cloud by yourself. +**Notice：** We do not consider or plan to release any unilateral test results based on this standard. For reference, you can purchase server tests on the cloud by yourself. This repository houses user-friendly, cloud-ready benchmarking suites for the following messaging platforms: @@ -14,7 +15,7 @@ This repository houses user-friendly, cloud-ready benchmarking suites for the fo * Generic [JMS](https://javaee.github.io/jms-spec/) * [KoP (Kafka-on-Pulsar)](https://github.com/streamnative/kop) * [NATS Streaming](https://nats.io/) -* [NSQ](https://nsq.io) +* [NSQ](https://nsq.io) * [Pravega](https://pravega.io/) * [RabbitMQ](https://www.rabbitmq.com/) * [Redis](https://redis.com/) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index 997a5ab45..1c6b2b8bf 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - benchmark-framework + benchmark-framework - - 9.4.42.v20210604 - + + 9.4.42.v20210604 + - - - ${project.groupId} - driver-api - ${project.version} - - - org.hdrhistogram - HdrHistogram - 2.1.10 - - - com.beust - jcommander - - - ${project.groupId} - driver-pulsar - ${project.version} - - - ${project.groupId} - driver-jms - ${project.version} - - - ${project.groupId} - driver-kafka - ${project.version} - - - ${project.groupId} - driver-kop - ${project.version} - - - ${project.groupId} - driver-pravega - ${project.version} - - - ${project.groupId} - driver-rocketmq - ${project.version} - - - ${project.groupId} - driver-rabbitmq - ${project.version} - - - ${project.groupId} - driver-artemis - ${project.version} - - - ${project.groupId} - driver-bookkeeper - ${project.version} - - - ${project.groupId} - driver-nats - ${project.version} - - - ${project.groupId} - driver-nats-streaming - ${project.version} - - - ${project.groupId} - driver-nsq - ${project.version} - - - ${project.groupId} - driver-redis - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${bookkeeper.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - - + + + ${project.groupId} + driver-api + ${project.version} + + + ${project.groupId} + driver-artemis + ${project.version} + + + ${project.groupId} + driver-bookkeeper + ${project.version} + + + ${project.groupId} + driver-jms + ${project.version} + + + ${project.groupId} + driver-kafka + ${project.version} + + + ${project.groupId} + driver-kop + ${project.version} + + + ${project.groupId} + driver-nats + ${project.version} + + + ${project.groupId} + driver-nats-streaming + ${project.version} + + + ${project.groupId} + driver-nsq + ${project.version} + + + ${project.groupId} + driver-pravega + ${project.version} + + + ${project.groupId} + driver-pulsar + ${project.version} + + + ${project.groupId} + driver-rabbitmq + ${project.version} + + + ${project.groupId} + driver-redis + ${project.version} + + + ${project.groupId} + driver-rocketmq + ${project.version} + + + com.beust + jcommander + + io.javalin javalin 1.3.0 - - org.asynchttpclient - async-http-client - 2.12.3 - - - - org.eclipse.jetty - jetty-server - ${jetty.version} - - - org.eclipse.jetty - jetty-util - ${jetty.version} - - - org.assertj - assertj-core - - - org.junit.jupiter - junit-jupiter - - - org.mockito - mockito-junit-jupiter - - + + org.apache.bookkeeper.stats + prometheus-metrics-provider + ${bookkeeper.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.asynchttpclient + async-http-client + 2.12.3 + + + org.asynchttpclient + async-http-client + 2.12.3 + + + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-util + ${jetty.version} + + + org.hdrhistogram + HdrHistogram + 2.1.10 + + + org.assertj + assertj-core + test + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-junit-jupiter + test + + - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - build-classpath - generate-sources - - build-classpath - - - target/classpath.txt - - - - - - + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + build-classpath + + build-classpath + + generate-sources + + target/classpath.txt + + + + + + diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index 8d0728098..25c5c5a9f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -13,16 +13,6 @@ */ package io.openmessaging.benchmark; -import java.io.File; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -31,40 +21,62 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - import io.openmessaging.benchmark.worker.DistributedWorkersEnsemble; import io.openmessaging.benchmark.worker.LocalWorker; import io.openmessaging.benchmark.worker.Worker; +import java.io.File; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Benchmark { static class Arguments { - @Parameter(names = {"-c", "--csv"}, description = "Print results from this directory to a csv file") + @Parameter( + names = {"-c", "--csv"}, + description = "Print results from this directory to a csv file") String resultsDir; - @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) + @Parameter( + names = {"-h", "--help"}, + description = "Help message", + help = true) boolean help; - @Parameter(names = { "-d", - "--drivers" }, description = "Drivers list. eg.: pulsar/pulsar.yaml,kafka/kafka.yaml")//, required = true) + @Parameter( + names = {"-d", "--drivers"}, + description = + "Drivers list. eg.: pulsar/pulsar.yaml,kafka/kafka.yaml") // , required = true) public List drivers; - @Parameter(names = { "-w", - "--workers" }, description = "List of worker nodes. eg: http://1.2.3.4:8080,http://4.5.6.7:8080") + @Parameter( + names = {"-w", "--workers"}, + description = "List of worker nodes. eg: http://1.2.3.4:8080,http://4.5.6.7:8080") public List workers; - @Parameter(names = { "-wf", - "--workers-file" }, description = "Path to a YAML file containing the list of workers addresses") + @Parameter( + names = {"-wf", "--workers-file"}, + description = "Path to a YAML file containing the list of workers addresses") public File workersFile; - @Parameter(names = { "-x", "--extra" }, description = "Allocate extra consumer workers when your backlog builds.") + @Parameter( + names = {"-x", "--extra"}, + description = "Allocate extra consumer workers when your backlog builds.") boolean extraConsumers; - @Parameter(description = "Workloads")//, required = true) + @Parameter(description = "Workloads") // , required = true) public List workloads; - @Parameter(names = { "-o", "--output" }, description = "Output", required = false) + @Parameter( + names = {"-o", "--output"}, + description = "Output", + required = false) public String output; } @@ -86,7 +98,7 @@ public static void main(String[] args) throws Exception { System.exit(-1); } - if(arguments.resultsDir != null) { + if (arguments.resultsDir != null) { ResultsToCsv r = new ResultsToCsv(); r.writeAllResultFiles(arguments.resultsDir); System.exit(0); @@ -132,51 +144,67 @@ public static void main(String[] args) throws Exception { worker = new LocalWorker(); } - workloads.forEach((workloadName, workload) -> { - arguments.drivers.forEach(driverConfig -> { - try { - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); - File driverConfigFile = new File(driverConfig); - DriverConfiguration driverConfiguration = mapper.readValue(driverConfigFile, - DriverConfiguration.class); - log.info("--------------- WORKLOAD : {} --- DRIVER : {}---------------", workload.name, - driverConfiguration.name); - - // Stop any left over workload - worker.stopAll(); - - worker.initializeDriver(new File(driverConfig)); - - WorkloadGenerator generator = new WorkloadGenerator(driverConfiguration.name, workload, worker); - - TestResult result = generator.run(); - - boolean useOutput = (arguments.output != null) && (arguments.output.length() > 0); - - String fileName = useOutput? arguments.output: String.format("%s-%s-%s.json", workloadName, - driverConfiguration.name, dateFormat.format(new Date())); - - log.info("Writing test result into {}/{}", workloadName, fileName); - File folder = new File(workloadName); - if (!folder.mkdirs()) { - log.debug("Unable to create folder {}", folder); - } - writer.writeValue(new File(folder, fileName), result); - - generator.close(); - } catch (Exception e) { - log.error("Failed to run the workload '{}' for driver '{}'", workload.name, driverConfig, e); - } finally { - worker.stopAll(); - } - }); - }); + workloads.forEach( + (workloadName, workload) -> { + arguments.drivers.forEach( + driverConfig -> { + try { + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); + File driverConfigFile = new File(driverConfig); + DriverConfiguration driverConfiguration = + mapper.readValue(driverConfigFile, DriverConfiguration.class); + log.info( + "--------------- WORKLOAD : {} --- DRIVER : {}---------------", + workload.name, + driverConfiguration.name); + + // Stop any left over workload + worker.stopAll(); + + worker.initializeDriver(new File(driverConfig)); + + WorkloadGenerator generator = + new WorkloadGenerator(driverConfiguration.name, workload, worker); + + TestResult result = generator.run(); + + boolean useOutput = (arguments.output != null) && (arguments.output.length() > 0); + + String fileName = + useOutput + ? arguments.output + : String.format( + "%s-%s-%s.json", + workloadName, + driverConfiguration.name, + dateFormat.format(new Date())); + + log.info("Writing test result into {}/{}", workloadName, fileName); + File folder = new File(workloadName); + if (!folder.mkdirs()) { + log.debug("Unable to create folder {}", folder); + } + writer.writeValue(new File(folder, fileName), result); + + generator.close(); + } catch (Exception e) { + log.error( + "Failed to run the workload '{}' for driver '{}'", + workload.name, + driverConfig, + e); + } finally { + worker.stopAll(); + } + }); + }); worker.close(); } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); static { mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java index e220cfa2b..8e6c858a3 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/DriverConfiguration.java @@ -17,5 +17,4 @@ public class DriverConfiguration { public String name; public String driverClass; - } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java index c1133a793..9ee84ece3 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/ResultsToCsv.java @@ -14,13 +14,11 @@ package io.openmessaging.benchmark; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.HdrHistogram.Histogram; -import org.bouncycastle.util.test.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.*; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.text.MessageFormat; import java.time.Instant; import java.util.ArrayList; @@ -28,6 +26,9 @@ import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; +import org.HdrHistogram.Histogram; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ResultsToCsv { @@ -43,24 +44,29 @@ public void writeAllResultFiles(String directory) { Arrays.sort(directoryListing); List lines = new ArrayList<>(); - lines.add("topics,partitions,message-size,producers-per-topic,consumers-per-topic," + - "prod-rate-min,prod-rate-avg,prod-rate-std-dev,prod-rate-max," + - "con-rate-min,con-rate-avg,con-rate-std-dev,con-rate-max,"); + lines.add( + "topics,partitions,message-size,producers-per-topic,consumers-per-topic," + + "prod-rate-min,prod-rate-avg,prod-rate-std-dev,prod-rate-max," + + "con-rate-min,con-rate-avg,con-rate-std-dev,con-rate-max,"); List results = new ArrayList<>(); for (File file : directoryListing) { if (file.isFile() && file.getAbsolutePath().endsWith(".json")) { ObjectMapper objectMapper = new ObjectMapper(); - TestResult tr = objectMapper.readValue(new File(file.getAbsolutePath()), TestResult.class); + TestResult tr = + objectMapper.readValue(new File(file.getAbsolutePath()), TestResult.class); results.add(tr); } } - List sortedResults = results.stream().sorted( - Comparator.comparing(TestResult::getMessageSize) - .thenComparing(TestResult::getTopics) - .thenComparing(TestResult::getPartitions)).collect(Collectors.toList()); - for(TestResult tr : sortedResults) { + List sortedResults = + results.stream() + .sorted( + Comparator.comparing(TestResult::getMessageSize) + .thenComparing(TestResult::getTopics) + .thenComparing(TestResult::getPartitions)) + .collect(Collectors.toList()); + for (TestResult tr : sortedResults) { lines.add(extractResults(tr)); } @@ -71,8 +77,7 @@ public void writeAllResultFiles(String directory) { } log.info("Results extracted into CSV " + resultsFileName); } - } - catch(IOException e) { + } catch (IOException e) { log.error("Failed creating csv file.", e); } } @@ -82,38 +87,37 @@ public String extractResults(TestResult tr) { Histogram prodRateHistogram = new Histogram(10000000, 1); Histogram conRateHistogram = new Histogram(10000000, 1); - for(Double rate : tr.publishRate) { + for (Double rate : tr.publishRate) { prodRateHistogram.recordValueWithCount(rate.longValue(), 2); } - for(Double rate : tr.consumeRate) { + for (Double rate : tr.consumeRate) { conRateHistogram.recordValueWithCount(rate.longValue(), 2); } - String line = MessageFormat.format("{0,number,#},{1,number,#},{2,number,#},{3,number,#},{4,number,#}," + - "{5,number,#},{6,number,#},{7,number,#.##},{8,number,#}," + - "{9,number,#},{10,number,#},{11,number,#.##},{12,number,#}", - tr.topics, - tr.partitions, - tr.messageSize, - tr.producersPerTopic, - tr.consumersPerTopic, - prodRateHistogram.getMinNonZeroValue(), - prodRateHistogram.getMean(), - prodRateHistogram.getStdDeviation(), - prodRateHistogram.getMaxValue(), - conRateHistogram.getMinNonZeroValue(), - conRateHistogram.getMean(), - conRateHistogram.getStdDeviation(), - conRateHistogram.getMaxValue()); + String line = + MessageFormat.format( + "{0,number,#},{1,number,#},{2,number,#},{3,number,#},{4,number,#}," + + "{5,number,#},{6,number,#},{7,number,#.##},{8,number,#}," + + "{9,number,#},{10,number,#},{11,number,#.##},{12,number,#}", + tr.topics, + tr.partitions, + tr.messageSize, + tr.producersPerTopic, + tr.consumersPerTopic, + prodRateHistogram.getMinNonZeroValue(), + prodRateHistogram.getMean(), + prodRateHistogram.getStdDeviation(), + prodRateHistogram.getMaxValue(), + conRateHistogram.getMinNonZeroValue(), + conRateHistogram.getMean(), + conRateHistogram.getStdDeviation(), + conRateHistogram.getMaxValue()); return line; - } - catch(Exception e) { + } catch (Exception e) { log.error("Error writing results csv", e); throw new RuntimeException(e); } } - - } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java index 88a11c0e2..fc9781b4d 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/TestResult.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark; + import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java index 52da2c530..fbc16e613 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workers.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark; + import java.util.ArrayList; import java.util.List; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java index 759eb09af..c437b187e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java @@ -13,15 +13,16 @@ */ package io.openmessaging.benchmark; + import io.openmessaging.benchmark.utils.distributor.KeyDistributorType; public class Workload { public String name; - /** Number of topics to create in the test */ + /** Number of topics to create in the test. */ public int topics; - /** Number of partitions each topic will contain */ + /** Number of partitions each topic will contain. */ public int partitionsPerTopic; public KeyDistributorType keyDistributor = KeyDistributorType.NO_KEY; @@ -43,11 +44,11 @@ public class Workload { public int producerRate; /** - * If the consumer backlog is > 0, the generator will accumulate messages until the requested amount of storage is - * retained and then it will start the consumers to drain it. + * If the consumer backlog is > 0, the generator will accumulate messages until the requested + * amount of storage is retained and then it will start the consumers to drain it. * - * The testDurationMinutes will be overruled to allow the test to complete when the consumer has drained all the - * backlog and it's on par with the producer + *

The testDurationMinutes will be overruled to allow the test to complete when the consumer + * has drained all the backlog and it's on par with the producer */ public long consumerBacklogSizeGB = 0; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 122a4efb3..6905fdf22 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -13,23 +13,10 @@ */ package io.openmessaging.benchmark; -import io.openmessaging.benchmark.utils.RandomGenerator; -import java.io.IOException; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang.ArrayUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import io.netty.util.concurrent.DefaultThreadFactory; import io.openmessaging.benchmark.utils.PaddingDecimalFormat; +import io.openmessaging.benchmark.utils.RandomGenerator; import io.openmessaging.benchmark.utils.Timer; import io.openmessaging.benchmark.utils.payload.FilePayloadReader; import io.openmessaging.benchmark.utils.payload.PayloadReader; @@ -41,6 +28,18 @@ import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment; import io.openmessaging.benchmark.worker.commands.TopicSubscription; import io.openmessaging.benchmark.worker.commands.TopicsInfo; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class WorkloadGenerator implements AutoCloseable { @@ -48,8 +47,8 @@ public class WorkloadGenerator implements AutoCloseable { private final Workload workload; private final Worker worker; - private final ExecutorService executor = Executors - .newCachedThreadPool(new DefaultThreadFactory("messaging-benchmark")); + private final ExecutorService executor = + Executors.newCachedThreadPool(new DefaultThreadFactory("messaging-benchmark")); private volatile boolean runCompleted = false; private volatile boolean needToWaitForBacklogDraining = false; @@ -62,13 +61,15 @@ public WorkloadGenerator(String driverName, Workload workload, Worker worker) { this.worker = worker; if (workload.consumerBacklogSizeGB > 0 && workload.producerRate == 0) { - throw new IllegalArgumentException("Cannot probe producer sustainable rate when building backlog"); + throw new IllegalArgumentException( + "Cannot probe producer sustainable rate when building backlog"); } } public TestResult run() throws Exception { Timer timer = new Timer(); - List topics = worker.createTopics(new TopicsInfo(workload.topics, workload.partitionsPerTopic)); + List topics = + worker.createTopics(new TopicsInfo(workload.topics, workload.partitionsPerTopic)); log.info("Created {} topics in {} ms", topics.size(), timer.elapsedMillis()); createConsumers(topics); @@ -82,14 +83,15 @@ public TestResult run() throws Exception { // Producer rate is 0 and we need to discover the sustainable rate targetPublishRate = 10000; - executor.execute(() -> { - // Run background controller to adjust rate - try { - findMaximumSustainableRate(targetPublishRate); - } catch (IOException e) { - log.warn("Failure in finding max sustainable rate", e); - } - }); + executor.execute( + () -> { + // Run background controller to adjust rate + try { + findMaximumSustainableRate(targetPublishRate); + } catch (IOException e) { + log.warn("Failure in finding max sustainable rate", e); + } + }); } final PayloadReader payloadReader = new FilePayloadReader(workload.messageSize); @@ -99,21 +101,20 @@ public TestResult run() throws Exception { producerWorkAssignment.publishRate = targetPublishRate; producerWorkAssignment.payloadData = new ArrayList<>(); - if(workload.useRandomizedPayloads) { + if (workload.useRandomizedPayloads) { // create messages that are part random and part zeros // better for testing effects of compression Random r = new Random(); - int randomBytes = (int)(workload.messageSize * workload.randomBytesRatio); + int randomBytes = (int) (workload.messageSize * workload.randomBytesRatio); int zerodBytes = workload.messageSize - randomBytes; - for(int i = 0; i 0) { - executor.execute(() -> { - try { - buildAndDrainBacklog(topics); - } catch (IOException e) { - e.printStackTrace(); - } - }); + executor.execute( + () -> { + try { + buildAndDrainBacklog(topics); + } catch (IOException e) { + e.printStackTrace(); + } + }); } worker.resetStats(); @@ -146,7 +148,8 @@ public TestResult run() throws Exception { private void ensureTopicsAreReady() throws IOException { log.info("Waiting for consumers to be ready"); - // This is work around the fact that there's no way to have a consumer ready in Kafka without first publishing + // This is work around the fact that there's no way to have a consumer ready in Kafka without + // first publishing // some message on the topic, which will then trigger the partitions assignment to the consumers int expectedMessages = workload.topics * workload.subscriptionsPerTopic; @@ -154,12 +157,15 @@ private void ensureTopicsAreReady() throws IOException { // In this case we just publish 1 message and then wait for consumers to receive the data worker.probeProducers(); - long start = System.currentTimeMillis(); + long start = System.currentTimeMillis(); long end = start + 60 * 1000; while (System.currentTimeMillis() < end) { CountersStats stats = worker.getCountersStats(); - log.info("Waiting for topics to be ready -- Sent: {}, Received: {}", stats.messagesSent, stats.messagesReceived); + log.info( + "Waiting for topics to be ready -- Sent: {}, Received: {}", + stats.messagesSent, + stats.messagesReceived); if (stats.messagesReceived < expectedMessages) { try { Thread.sleep(2_000); @@ -173,16 +179,18 @@ private void ensureTopicsAreReady() throws IOException { if (System.currentTimeMillis() >= end) { throw new RuntimeException("Timed out waiting for consumers to be ready"); - } - else { + } else { log.info("All consumers are ready"); - } + } } /** - * Adjust the publish rate to a level that is sustainable, meaning that we can consume all the messages that are - * being produced + * Adjust the publish rate to a level that is sustainable, meaning that we can consume all the + * messages that are being produced. + * + * @param currentRate */ + @SuppressWarnings("checkstyle:LineLength") private void findMaximumSustainableRate(double currentRate) throws IOException { double maxRate = Double.MAX_VALUE; // Discovered max sustainable rate double minRate = 0.1; @@ -212,16 +220,24 @@ private void findMaximumSustainableRate(double currentRate) throws IOException { long totalMessagesReceived = stats.messagesReceived; long messagesPublishedInPeriod = totalMessagesSent - localTotalMessagesSentCounter; long messagesReceivedInPeriod = totalMessagesReceived - localTotalMessagesReceivedCounter; - double publishRateInLastPeriod = messagesPublishedInPeriod / (double) (currentTime - lastControlTimestamp) - * TimeUnit.SECONDS.toNanos(1); - double receiveRateInLastPeriod = messagesReceivedInPeriod / (double) (currentTime - lastControlTimestamp) - * TimeUnit.SECONDS.toNanos(1); + double publishRateInLastPeriod = + messagesPublishedInPeriod + / (double) (currentTime - lastControlTimestamp) + * TimeUnit.SECONDS.toNanos(1); + double receiveRateInLastPeriod = + messagesReceivedInPeriod + / (double) (currentTime - lastControlTimestamp) + * TimeUnit.SECONDS.toNanos(1); if (log.isDebugEnabled()) { log.debug( "total-send: {} -- total-received: {} -- int-sent: {} -- int-received: {} -- sent-rate: {} -- received-rate: {}", - totalMessagesSent, totalMessagesReceived, messagesPublishedInPeriod, messagesReceivedInPeriod, - publishRateInLastPeriod, receiveRateInLastPeriod); + totalMessagesSent, + totalMessagesReceived, + messagesPublishedInPeriod, + messagesReceivedInPeriod, + publishRateInLastPeriod, + receiveRateInLastPeriod); } localTotalMessagesSentCounter = totalMessagesSent; @@ -229,9 +245,13 @@ private void findMaximumSustainableRate(double currentRate) throws IOException { lastControlTimestamp = currentTime; if (log.isDebugEnabled()) { - log.debug("Current rate: {} -- Publish rate {} -- Consume Rate: {} -- min-rate: {} -- max-rate: {}", - dec.format(currentRate), dec.format(publishRateInLastPeriod), - dec.format(receiveRateInLastPeriod), dec.format(minRate), dec.format(maxRate)); + log.debug( + "Current rate: {} -- Publish rate {} -- Consume Rate: {} -- min-rate: {} -- max-rate: {}", + dec.format(currentRate), + dec.format(publishRateInLastPeriod), + dec.format(receiveRateInLastPeriod), + dec.format(minRate), + dec.format(maxRate)); } if (publishRateInLastPeriod < currentRate * 0.95) { @@ -250,7 +270,8 @@ private void findMaximumSustainableRate(double currentRate) throws IOException { worker.adjustPublishRate(minRate / 10); while (true) { stats = worker.getCountersStats(); - long backlog = workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived; + long backlog = + workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived; if (backlog < 1000) { break; } @@ -299,12 +320,13 @@ public void close() throws Exception { private void createConsumers(List topics) throws IOException { ConsumerAssignment consumerAssignment = new ConsumerAssignment(); - for(String topic: topics){ - for(int i = 0; i < workload.subscriptionsPerTopic; i++){ - String subscriptionName = String.format("sub-%03d-%s", i, RandomGenerator.getRandomString()); + for (String topic : topics) { + for (int i = 0; i < workload.subscriptionsPerTopic; i++) { + String subscriptionName = + String.format("sub-%03d-%s", i, RandomGenerator.getRandomString()); for (int j = 0; j < workload.consumerPerSubscription; j++) { - consumerAssignment.topicsSubscriptions - .add(new TopicSubscription(topic, subscriptionName)); + consumerAssignment.topicsSubscriptions.add( + new TopicSubscription(topic, subscriptionName)); } } } @@ -314,7 +336,10 @@ private void createConsumers(List topics) throws IOException { Timer timer = new Timer(); worker.createConsumers(consumerAssignment); - log.info("Created {} consumers in {} ms", consumerAssignment.topicsSubscriptions.size(), timer.elapsedMillis()); + log.info( + "Created {} consumers in {} ms", + consumerAssignment.topicsSubscriptions.size(), + timer.elapsedMillis()); } private void createProducers(List topics) throws IOException { @@ -344,8 +369,9 @@ private void buildAndDrainBacklog(List topics) throws IOException { while (true) { CountersStats stats = worker.getCountersStats(); - long currentBacklogSize = (workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived) - * workload.messageSize; + long currentBacklogSize = + (workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived) + * workload.messageSize; if (currentBacklogSize >= requestedBacklogSize) { break; @@ -368,7 +394,8 @@ private void buildAndDrainBacklog(List topics) throws IOException { while (true) { CountersStats stats = worker.getCountersStats(); - long currentBacklog = workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived; + long currentBacklog = + workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived; if (currentBacklog <= minBacklog) { log.info("--- Completed backlog draining in {} s ---", timer.elapsedSeconds()); needToWaitForBacklogDraining = false; @@ -383,6 +410,7 @@ private void buildAndDrainBacklog(List topics) throws IOException { } } + @SuppressWarnings({"checkstyle:LineLength", "checkstyle:MethodLength"}) private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throws IOException { long startTime = System.nanoTime(); @@ -419,14 +447,19 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw double consumeRate = stats.messagesReceived / elapsed; double consumeThroughput = stats.bytesReceived / elapsed / 1024 / 1024; - long currentBacklog = Math.max(0L, workload.subscriptionsPerTopic * stats.totalMessagesSent - - stats.totalMessagesReceived); + long currentBacklog = + Math.max( + 0L, + workload.subscriptionsPerTopic * stats.totalMessagesSent + - stats.totalMessagesReceived); log.info( "Pub rate {} msg/s / {} MB/s | Pub err {} err/s | Cons rate {} msg/s / {} MB/s | Backlog: {} K | Pub Latency (ms) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {} | Pub Delay Latency (us) avg: {} - 50%: {} - 99%: {} - 99.9%: {} - Max: {}", - rateFormat.format(publishRate), throughputFormat.format(publishThroughput), + rateFormat.format(publishRate), + throughputFormat.format(publishThroughput), rateFormat.format(errorRate), - rateFormat.format(consumeRate), throughputFormat.format(consumeThroughput), + rateFormat.format(consumeRate), + throughputFormat.format(consumeThroughput), dec.format(currentBacklog / 1000.0), // dec.format(microsToMillis(stats.publishLatency.getMean())), dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(50))), @@ -448,8 +481,10 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw result.publishLatency75pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(75))); result.publishLatency95pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(95))); result.publishLatency99pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99))); - result.publishLatency999pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99.9))); - result.publishLatency9999pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99.99))); + result.publishLatency999pct.add( + microsToMillis(stats.publishLatency.getValueAtPercentile(99.9))); + result.publishLatency9999pct.add( + microsToMillis(stats.publishLatency.getValueAtPercentile(99.99))); result.publishLatencyMax.add(microsToMillis(stats.publishLatency.getMaxValue())); result.publishDelayLatencyAvg.add(stats.publishDelayLatency.getMean()); @@ -461,14 +496,19 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw result.publishDelayLatency9999pct.add(stats.publishDelayLatency.getValueAtPercentile(99.99)); result.publishDelayLatencyMax.add(stats.publishDelayLatency.getMaxValue()); - result.endToEndLatencyAvg.add(microsToMillis(stats.endToEndLatency.getMean())); - result.endToEndLatency50pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(50))); - result.endToEndLatency75pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(75))); - result.endToEndLatency95pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(95))); - result.endToEndLatency99pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99))); - result.endToEndLatency999pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.9))); - result.endToEndLatency9999pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.99))); + result.endToEndLatency50pct.add( + microsToMillis(stats.endToEndLatency.getValueAtPercentile(50))); + result.endToEndLatency75pct.add( + microsToMillis(stats.endToEndLatency.getValueAtPercentile(75))); + result.endToEndLatency95pct.add( + microsToMillis(stats.endToEndLatency.getValueAtPercentile(95))); + result.endToEndLatency99pct.add( + microsToMillis(stats.endToEndLatency.getValueAtPercentile(99))); + result.endToEndLatency999pct.add( + microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.9))); + result.endToEndLatency9999pct.add( + microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.99))); result.endToEndLatencyMax.add(microsToMillis(stats.endToEndLatency.getMaxValue())); if (now >= testEndTime && !needToWaitForBacklogDraining) { @@ -495,42 +535,65 @@ private TestResult printAndCollectStats(long testDurations, TimeUnit unit) throw result.aggregatedPublishLatency75pct = agg.publishLatency.getValueAtPercentile(75) / 1000.0; result.aggregatedPublishLatency95pct = agg.publishLatency.getValueAtPercentile(95) / 1000.0; result.aggregatedPublishLatency99pct = agg.publishLatency.getValueAtPercentile(99) / 1000.0; - result.aggregatedPublishLatency999pct = agg.publishLatency.getValueAtPercentile(99.9) / 1000.0; - result.aggregatedPublishLatency9999pct = agg.publishLatency.getValueAtPercentile(99.99) / 1000.0; + result.aggregatedPublishLatency999pct = + agg.publishLatency.getValueAtPercentile(99.9) / 1000.0; + result.aggregatedPublishLatency9999pct = + agg.publishLatency.getValueAtPercentile(99.99) / 1000.0; result.aggregatedPublishLatencyMax = agg.publishLatency.getMaxValue() / 1000.0; result.aggregatedPublishDelayLatencyAvg = agg.publishDelayLatency.getMean(); - result.aggregatedPublishDelayLatency50pct = agg.publishDelayLatency.getValueAtPercentile(50); - result.aggregatedPublishDelayLatency75pct = agg.publishDelayLatency.getValueAtPercentile(75); - result.aggregatedPublishDelayLatency95pct = agg.publishDelayLatency.getValueAtPercentile(95); - result.aggregatedPublishDelayLatency99pct = agg.publishDelayLatency.getValueAtPercentile(99); - result.aggregatedPublishDelayLatency999pct = agg.publishDelayLatency.getValueAtPercentile(99.9); - result.aggregatedPublishDelayLatency9999pct = agg.publishDelayLatency.getValueAtPercentile(99.99); + result.aggregatedPublishDelayLatency50pct = + agg.publishDelayLatency.getValueAtPercentile(50); + result.aggregatedPublishDelayLatency75pct = + agg.publishDelayLatency.getValueAtPercentile(75); + result.aggregatedPublishDelayLatency95pct = + agg.publishDelayLatency.getValueAtPercentile(95); + result.aggregatedPublishDelayLatency99pct = + agg.publishDelayLatency.getValueAtPercentile(99); + result.aggregatedPublishDelayLatency999pct = + agg.publishDelayLatency.getValueAtPercentile(99.9); + result.aggregatedPublishDelayLatency9999pct = + agg.publishDelayLatency.getValueAtPercentile(99.99); result.aggregatedPublishDelayLatencyMax = agg.publishDelayLatency.getMaxValue(); - result.aggregatedEndToEndLatencyAvg = agg.endToEndLatency.getMean() / 1000.0; - result.aggregatedEndToEndLatency50pct = agg.endToEndLatency.getValueAtPercentile(50) / 1000.0; - result.aggregatedEndToEndLatency75pct = agg.endToEndLatency.getValueAtPercentile(75) / 1000.0; - result.aggregatedEndToEndLatency95pct = agg.endToEndLatency.getValueAtPercentile(95) / 1000.0; - result.aggregatedEndToEndLatency99pct = agg.endToEndLatency.getValueAtPercentile(99) / 1000.0; - result.aggregatedEndToEndLatency999pct = agg.endToEndLatency.getValueAtPercentile(99.9) / 1000.0; - result.aggregatedEndToEndLatency9999pct = agg.endToEndLatency.getValueAtPercentile(99.99) / 1000.0; - result.aggregatedEndToEndLatencyMax = agg.endToEndLatency.getMaxValue() / 1000.0; - - agg.publishLatency.percentiles(100).forEach(value -> { - result.aggregatedPublishLatencyQuantiles.put(value.getPercentile(), - value.getValueIteratedTo() / 1000.0); - }); - - agg.publishDelayLatency.percentiles(100).forEach(value -> { - result.aggregatedPublishDelayLatencyQuantiles.put(value.getPercentile(), - value.getValueIteratedTo()); - }); - - agg.endToEndLatency.percentiles(100).forEach(value -> { - result.aggregatedEndToEndLatencyQuantiles.put(value.getPercentile(), - microsToMillis(value.getValueIteratedTo())); - }); + result.aggregatedEndToEndLatencyAvg = agg.endToEndLatency.getMean() / 1000.0; + result.aggregatedEndToEndLatency50pct = + agg.endToEndLatency.getValueAtPercentile(50) / 1000.0; + result.aggregatedEndToEndLatency75pct = + agg.endToEndLatency.getValueAtPercentile(75) / 1000.0; + result.aggregatedEndToEndLatency95pct = + agg.endToEndLatency.getValueAtPercentile(95) / 1000.0; + result.aggregatedEndToEndLatency99pct = + agg.endToEndLatency.getValueAtPercentile(99) / 1000.0; + result.aggregatedEndToEndLatency999pct = + agg.endToEndLatency.getValueAtPercentile(99.9) / 1000.0; + result.aggregatedEndToEndLatency9999pct = + agg.endToEndLatency.getValueAtPercentile(99.99) / 1000.0; + result.aggregatedEndToEndLatencyMax = agg.endToEndLatency.getMaxValue() / 1000.0; + + agg.publishLatency + .percentiles(100) + .forEach( + value -> { + result.aggregatedPublishLatencyQuantiles.put( + value.getPercentile(), value.getValueIteratedTo() / 1000.0); + }); + + agg.publishDelayLatency + .percentiles(100) + .forEach( + value -> { + result.aggregatedPublishDelayLatencyQuantiles.put( + value.getPercentile(), value.getValueIteratedTo()); + }); + + agg.endToEndLatency + .percentiles(100) + .forEach( + value -> { + result.aggregatedEndToEndLatencyQuantiles.put( + value.getPercentile(), microsToMillis(value.getValueIteratedTo())); + }); break; } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java index 2af6cfca9..01a1ae51e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/ListPartition.java @@ -13,18 +13,19 @@ */ package io.openmessaging.benchmark.utils; + import java.util.ArrayList; import java.util.List; public class ListPartition { /** - * partition a list to specified size + * partition a list to specified size. * * @param originList * @param size * @param - * @return + * @return the partitioned list */ public static List> partitionList(List originList, int size) { @@ -55,5 +56,4 @@ public static List> partitionList(List originList, int size) { } return resultList; } - } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java index 1d9d91f18..a598ddb2a 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/PaddingDecimalFormat.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.utils; + import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.text.FieldPosition; @@ -22,8 +23,11 @@ public class PaddingDecimalFormat extends DecimalFormat { private int minimumLength; /** - * Creates a PaddingDecimalFormat using the given pattern and minimum minimumLength and the symbols for the default - * locale. + * Creates a PaddingDecimalFormat using the given pattern and minimum minimumLength and the + * symbols for the default locale. + * + * @param pattern + * @param minLength */ public PaddingDecimalFormat(String pattern, int minLength) { super(pattern); @@ -32,6 +36,10 @@ public PaddingDecimalFormat(String pattern, int minLength) { /** * Creates a PaddingDecimalFormat using the given pattern, symbols and minimum minimumLength. + * + * @param pattern + * @param symbols + * @param minLength */ public PaddingDecimalFormat(String pattern, DecimalFormatSymbols symbols, int minLength) { super(pattern, symbols); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java index 2eba74b9e..13c4f501c 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/RandomGenerator.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.utils; + import com.google.common.io.BaseEncoding; import java.util.Random; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java index 04144d679..11a49de44 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.utils; + import java.util.concurrent.TimeUnit; import java.util.function.Supplier; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java index 187cbf9c2..75ae74797 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java @@ -13,7 +13,7 @@ */ package io.openmessaging.benchmark.utils; -import java.time.Clock; + import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.locks.LockSupport; @@ -46,8 +46,8 @@ public final class UniformRateLimiter { this.opsPerSec = opsPerSec; intervalNs = Math.round(ONE_SEC_IN_NS / opsPerSec); this.nanoClock = nanoClock; - } + public UniformRateLimiter(final double opsPerSec) { this(opsPerSec, () -> System.nanoTime()); } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java index 7cf4d240f..14576c985 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributor.java @@ -26,8 +26,8 @@ */ package io.openmessaging.benchmark.utils.distributor; -import com.google.common.io.BaseEncoding; +import com.google.common.io.BaseEncoding; import java.util.Random; public abstract class KeyDistributor { @@ -69,8 +69,9 @@ public static KeyDistributor build(KeyDistributorType keyType) { case RANDOM_NANO: keyDistributor = new RandomNano(); break; + default: + throw new IllegalStateException("Unexpected KeyDistributorType: " + keyType); } return keyDistributor; } - } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java index e69d41d75..3cb1cf1c0 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyDistributorType.java @@ -26,22 +26,17 @@ */ package io.openmessaging.benchmark.utils.distributor; + import com.fasterxml.jackson.annotation.JsonEnumDefaultValue; public enum KeyDistributorType { + /** Key distributor that returns null keys to have default publish semantics. */ @JsonEnumDefaultValue - /** - * Key distributor that returns null keys to have default publish semantics - */ NO_KEY, - /** - * Genarate a finite number of "keys" and cycle through them in round-robin fashion - */ + /** Genarate a finite number of "keys" and cycle through them in round-robin fashion. */ KEY_ROUND_ROBIN, - /** - * Random distribution based on System.nanoTime() - */ + /** Random distribution based on System.nanoTime(). */ RANDOM_NANO, } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java index 0a3182221..bb600e2ca 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/KeyRoundRobin.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.utils.distributor; + import javax.annotation.concurrent.NotThreadSafe; @NotThreadSafe diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java index f7265c140..cedbf1b1a 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/distributor/RandomNano.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.utils.distributor; + import javax.annotation.concurrent.ThreadSafe; @ThreadSafe diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java index 27cd9463f..2fe60e052 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/payload/FilePayloadReader.java @@ -13,12 +13,12 @@ */ package io.openmessaging.benchmark.utils.payload; +import static java.nio.file.Files.readAllBytes; + import java.io.File; import java.io.IOException; import java.text.MessageFormat; -import static java.nio.file.Files.readAllBytes; - public class FilePayloadReader implements PayloadReader { private final int expectedLength; @@ -41,8 +41,10 @@ public byte[] load(String resourceName) { private void checkPayloadLength(byte[] payload) { if (expectedLength != payload.length) { - throw new PayloadException(MessageFormat.format("Payload length mismatch. Actual is: {0}, but expected: {1} ", - payload.length, expectedLength)); + throw new PayloadException( + MessageFormat.format( + "Payload length mismatch. Actual is: {0}, but expected: {1} ", + payload.length, expectedLength)); } } } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java index d6f9d52fa..8723384cf 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/BenchmarkWorker.java @@ -13,36 +13,40 @@ */ package io.openmessaging.benchmark.worker; -import org.apache.bookkeeper.stats.Stats; -import org.apache.bookkeeper.stats.StatsProvider; -import org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider; -import org.apache.commons.configuration.CompositeConfiguration; -import org.apache.commons.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; - import io.javalin.Javalin; +import org.apache.bookkeeper.stats.Stats; +import org.apache.bookkeeper.stats.StatsProvider; +import org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider; +import org.apache.commons.configuration.CompositeConfiguration; +import org.apache.commons.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * A benchmark worker that listen for tasks to perform - */ +/** A benchmark worker that listen for tasks to perform. */ public class BenchmarkWorker { static class Arguments { - @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) + @Parameter( + names = {"-h", "--help"}, + description = "Help message", + help = true) boolean help; - @Parameter(names = { "-p", "--port" }, description = "HTTP port to listen on") + @Parameter( + names = {"-p", "--port"}, + description = "HTTP port to listen on") public int httpPort = 8080; - @Parameter(names = { "-sp", "--stats-port" }, description = "Stats port to listen on") + @Parameter( + names = {"-sp", "--stats-port"}, + description = "Stats port to listen on") public int statsPort = 8081; } @@ -71,9 +75,8 @@ public static void main(String[] args) throws Exception { StatsProvider provider = Stats.get(); provider.start(conf); - Runtime.getRuntime().addShutdownHook(new Thread( - () -> provider.stop(), - "benchmark-worker-shutdown-thread")); + Runtime.getRuntime() + .addShutdownHook(new Thread(() -> provider.stop(), "benchmark-worker-shutdown-thread")); // Dump configuration variables log.info("Starting benchmark with config: {}", writer.writeValueAsString(arguments)); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index ad34ac808..a27628851 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -17,6 +17,22 @@ import static java.util.stream.Collectors.toList; import static org.asynchttpclient.Dsl.asyncHttpClient; +import com.beust.jcommander.internal.Maps; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; +import io.openmessaging.benchmark.utils.ListPartition; +import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; +import io.openmessaging.benchmark.worker.commands.CountersStats; +import io.openmessaging.benchmark.worker.commands.CumulativeLatencies; +import io.openmessaging.benchmark.worker.commands.PeriodStats; +import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment; +import io.openmessaging.benchmark.worker.commands.TopicSubscription; +import io.openmessaging.benchmark.worker.commands.TopicsInfo; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -28,7 +44,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import java.util.zip.DataFormatException; - import org.HdrHistogram.Histogram; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.DefaultAsyncHttpClientConfig; @@ -36,24 +51,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.beust.jcommander.internal.Maps; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -import io.netty.buffer.ByteBufUtil; -import io.netty.buffer.Unpooled; -import io.openmessaging.benchmark.utils.ListPartition; -import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; -import io.openmessaging.benchmark.worker.commands.CountersStats; -import io.openmessaging.benchmark.worker.commands.CumulativeLatencies; -import io.openmessaging.benchmark.worker.commands.PeriodStats; -import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment; -import io.openmessaging.benchmark.worker.commands.TopicSubscription; -import io.openmessaging.benchmark.worker.commands.TopicsInfo; - public class DistributedWorkersEnsemble implements Worker { private final Thread shutdownHook = new Thread(this::stopAll); private final List workers; @@ -66,16 +63,17 @@ public class DistributedWorkersEnsemble implements Worker { public DistributedWorkersEnsemble(List workers, boolean extraConsumerWorkers) { Preconditions.checkArgument(workers.size() > 1); - DefaultAsyncHttpClientConfig.Builder clientBuilder = Dsl.config() - .setReadTimeout(600000) - .setRequestTimeout(600000); + DefaultAsyncHttpClientConfig.Builder clientBuilder = + Dsl.config().setReadTimeout(600000).setRequestTimeout(600000); httpClient = asyncHttpClient(clientBuilder); this.workers = unmodifiableList(workers); // For driver-jms extra consumers are required. // If there is an odd number of workers then allocate the extra to consumption. - int numberOfProducerWorkers = extraConsumerWorkers ? (workers.size() + 2) / 3 : workers.size() / 2; - List> partitions = Lists.partition(Lists.reverse(workers), workers.size() - numberOfProducerWorkers); + int numberOfProducerWorkers = + extraConsumerWorkers ? (workers.size() + 2) / 3 : workers.size() / 2; + List> partitions = + Lists.partition(Lists.reverse(workers), workers.size() - numberOfProducerWorkers); this.producerWorkers = partitions.get(1); this.consumerWorkers = partitions.get(0); @@ -95,14 +93,15 @@ public void initializeDriver(File configurationFile) throws IOException { @SuppressWarnings("unchecked") public List createTopics(TopicsInfo topicsInfo) throws IOException { // Create all topics from a single worker node - return (List) post(workers.get(0), "/create-topics", writer.writeValueAsBytes(topicsInfo), List.class) - .join(); + return (List) + post(workers.get(0), "/create-topics", writer.writeValueAsBytes(topicsInfo), List.class) + .join(); } @Override public void createProducers(List topics) { - List> topicsPerProducer = ListPartition.partitionList(topics, - producerWorkers.size()); + List> topicsPerProducer = + ListPartition.partitionList(topics, producerWorkers.size()); Map> topicsPerProducerMap = Maps.newHashMap(); int i = 0; for (List assignedTopics : topicsPerProducer) { @@ -110,19 +109,27 @@ public void createProducers(List topics) { } // Number of actually used workers might be less than available workers - numberOfUsedProducerWorkers = (int) topicsPerProducerMap.values().stream().filter(t -> !t.isEmpty()).count(); - log.debug("Producing worker count: {} of {}", numberOfUsedProducerWorkers, producerWorkers.size()); - - CompletableFuture[] futures = topicsPerProducerMap.keySet().stream().map(producer -> { - try { - return sendPost(producer, "/create-producers", - writer.writeValueAsBytes(topicsPerProducerMap.get(producer))); - } catch (Exception e) { - CompletableFuture future = new CompletableFuture<>(); - future.completeExceptionally(e); - return future; - } - }).toArray(this::newArray); + numberOfUsedProducerWorkers = + (int) topicsPerProducerMap.values().stream().filter(t -> !t.isEmpty()).count(); + log.debug( + "Producing worker count: {} of {}", numberOfUsedProducerWorkers, producerWorkers.size()); + + CompletableFuture[] futures = + topicsPerProducerMap.keySet().stream() + .map( + producer -> { + try { + return sendPost( + producer, + "/create-producers", + writer.writeValueAsBytes(topicsPerProducerMap.get(producer))); + } catch (Exception e) { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(e); + return future; + } + }) + .toArray(this::newArray); CompletableFuture.allOf(futures).join(); } @@ -131,7 +138,8 @@ public void createProducers(List topics) { public void startLoad(ProducerWorkAssignment producerWorkAssignment) throws IOException { // Reduce the publish rate across all the brokers producerWorkAssignment.publishRate /= numberOfUsedProducerWorkers; - log.debug("Setting worker assigned publish rate to {} msgs/sec", producerWorkAssignment.publishRate); + log.debug( + "Setting worker assigned publish rate to {} msgs/sec", producerWorkAssignment.publishRate); sendPost(producerWorkers, "/start-load", writer.writeValueAsBytes(producerWorkAssignment)); } @@ -165,9 +173,9 @@ public void resumeConsumers() throws IOException { @Override public void createConsumers(ConsumerAssignment overallConsumerAssignment) { - List> subscriptionsPerConsumer = ListPartition.partitionList( - overallConsumerAssignment.topicsSubscriptions, - consumerWorkers.size()); + List> subscriptionsPerConsumer = + ListPartition.partitionList( + overallConsumerAssignment.topicsSubscriptions, consumerWorkers.size()); Map topicsPerWorkerMap = Maps.newHashMap(); int i = 0; for (List tsl : subscriptionsPerConsumer) { @@ -176,16 +184,22 @@ public void createConsumers(ConsumerAssignment overallConsumerAssignment) { topicsPerWorkerMap.put(consumerWorkers.get(i++), individualAssignement); } - CompletableFuture[] futures = topicsPerWorkerMap.keySet().stream().map(consumer -> { - try { - return sendPost(consumer, "/create-consumers", - writer.writeValueAsBytes(topicsPerWorkerMap.get(consumer))); - } catch (Exception e) { - CompletableFuture future = new CompletableFuture<>(); - future.completeExceptionally(e); - return future; - } - }).toArray(this::newArray); + CompletableFuture[] futures = + topicsPerWorkerMap.keySet().stream() + .map( + consumer -> { + try { + return sendPost( + consumer, + "/create-consumers", + writer.writeValueAsBytes(topicsPerWorkerMap.get(consumer))); + } catch (Exception e) { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(e); + return future; + } + }) + .toArray(this::newArray); CompletableFuture.allOf(futures).join(); } @@ -194,69 +208,80 @@ public void createConsumers(ConsumerAssignment overallConsumerAssignment) { public PeriodStats getPeriodStats() { List individualStats = get(workers, "/period-stats", PeriodStats.class); PeriodStats stats = new PeriodStats(); - individualStats.forEach(is -> { - stats.messagesSent += is.messagesSent; - stats.messageSendErrors += is.messageSendErrors; - stats.bytesSent += is.bytesSent; - stats.messagesReceived += is.messagesReceived; - stats.bytesReceived += is.bytesReceived; - stats.totalMessagesSent += is.totalMessagesSent; - stats.totalMessageSendErrors += is.totalMessageSendErrors; - stats.totalMessagesReceived += is.totalMessagesReceived; - - try { - stats.publishLatency.add(Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30))); - - stats.publishDelayLatency.add(Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.publishDelayLatencyBytes), TimeUnit.SECONDS.toMicros(30))); - - stats.endToEndLatency.add(Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); - } catch (ArrayIndexOutOfBoundsException | DataFormatException e) { - throw new RuntimeException(e); - } - }); + individualStats.forEach( + is -> { + stats.messagesSent += is.messagesSent; + stats.messageSendErrors += is.messageSendErrors; + stats.bytesSent += is.bytesSent; + stats.messagesReceived += is.messagesReceived; + stats.bytesReceived += is.bytesReceived; + stats.totalMessagesSent += is.totalMessagesSent; + stats.totalMessageSendErrors += is.totalMessageSendErrors; + stats.totalMessagesReceived += is.totalMessagesReceived; + + try { + stats.publishLatency.add( + Histogram.decodeFromCompressedByteBuffer( + ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30))); + + stats.publishDelayLatency.add( + Histogram.decodeFromCompressedByteBuffer( + ByteBuffer.wrap(is.publishDelayLatencyBytes), TimeUnit.SECONDS.toMicros(30))); + + stats.endToEndLatency.add( + Histogram.decodeFromCompressedByteBuffer( + ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); + } catch (ArrayIndexOutOfBoundsException | DataFormatException e) { + throw new RuntimeException(e); + } + }); return stats; } @Override public CumulativeLatencies getCumulativeLatencies() { - List individualStats = get(workers, "/cumulative-latencies", CumulativeLatencies.class); + List individualStats = + get(workers, "/cumulative-latencies", CumulativeLatencies.class); CumulativeLatencies stats = new CumulativeLatencies(); - individualStats.forEach(is -> { - try { - stats.publishLatency.add(Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30))); - } catch (Exception e) { - log.error("Failed to decode publish latency: {}", - ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.publishLatencyBytes))); - throw new RuntimeException(e); - } - - try { - stats.publishDelayLatency.add(Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.publishDelayLatencyBytes), TimeUnit.SECONDS.toMicros(30))); - } catch (Exception e) { - log.error("Failed to decode publish delay latency: {}", - ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.publishDelayLatencyBytes))); - throw new RuntimeException(e); - } - - try { - stats.endToEndLatency.add(Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); - } catch (Exception e) { - log.error("Failed to decode end-to-end latency: {}", - ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.endToEndLatencyBytes))); - throw new RuntimeException(e); - } - }); + individualStats.forEach( + is -> { + try { + stats.publishLatency.add( + Histogram.decodeFromCompressedByteBuffer( + ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30))); + } catch (Exception e) { + log.error( + "Failed to decode publish latency: {}", + ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.publishLatencyBytes))); + throw new RuntimeException(e); + } + + try { + stats.publishDelayLatency.add( + Histogram.decodeFromCompressedByteBuffer( + ByteBuffer.wrap(is.publishDelayLatencyBytes), TimeUnit.SECONDS.toMicros(30))); + } catch (Exception e) { + log.error( + "Failed to decode publish delay latency: {}", + ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.publishDelayLatencyBytes))); + throw new RuntimeException(e); + } + + try { + stats.endToEndLatency.add( + Histogram.decodeFromCompressedByteBuffer( + ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); + } catch (Exception e) { + log.error( + "Failed to decode end-to-end latency: {}", + ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.endToEndLatencyBytes))); + throw new RuntimeException(e); + } + }); return stats; - } @Override @@ -264,11 +289,12 @@ public CountersStats getCountersStats() throws IOException { List individualStats = get(workers, "/counters-stats", CountersStats.class); CountersStats stats = new CountersStats(); - individualStats.forEach(is -> { - stats.messagesSent += is.messagesSent; - stats.messagesReceived += is.messagesReceived; - stats.messageSendErrors += is.messageSendErrors; - }); + individualStats.forEach( + is -> { + stats.messagesSent += is.messagesSent; + stats.messagesReceived += is.messagesReceived; + stats.messageSendErrors += is.messageSendErrors; + }); return stats; } @@ -279,63 +305,103 @@ public void resetStats() throws IOException { } /** - * Send a request to multiple hosts and wait for all responses + * Send a request to multiple hosts and wait for all responses. + * + * @param hosts + * @param path + * @param body */ private void sendPost(List hosts, String path, byte[] body) { - CompletableFuture[] futures = hosts.stream().map(w -> sendPost(w, path, body)).toArray(this::newArray); + CompletableFuture[] futures = + hosts.stream().map(w -> sendPost(w, path, body)).toArray(this::newArray); CompletableFuture.allOf(futures).join(); } private CompletableFuture sendPost(String host, String path, byte[] body) { - return httpClient.preparePost(host + path).setBody(body).execute().toCompletableFuture().thenApply(x -> { - if (x.getStatusCode() != 200) { - log.error("Failed to do HTTP post request to {}{} -- code: {}", host, path, x.getStatusCode()); - } - Preconditions.checkArgument(x.getStatusCode() == 200); - return (Void) null; - }); + return httpClient + .preparePost(host + path) + .setBody(body) + .execute() + .toCompletableFuture() + .thenApply( + x -> { + if (x.getStatusCode() != 200) { + log.error( + "Failed to do HTTP post request to {}{} -- code: {}", + host, + path, + x.getStatusCode()); + } + Preconditions.checkArgument(x.getStatusCode() == 200); + return (Void) null; + }); } private List get(List hosts, String path, Class clazz) { - CompletableFuture[] futures = hosts.stream().map(w -> get(w, path, clazz)).toArray(this::newArray); + CompletableFuture[] futures = + hosts.stream().map(w -> get(w, path, clazz)).toArray(this::newArray); CompletableFuture> resultFuture = new CompletableFuture<>(); - CompletableFuture.allOf(futures).thenRun(() -> { - resultFuture.complete(Stream.of(futures).map(CompletableFuture::join).collect(toList())); - }).exceptionally(ex -> { - resultFuture.completeExceptionally(ex); - return null; - }); + CompletableFuture.allOf(futures) + .thenRun( + () -> { + resultFuture.complete( + Stream.of(futures).map(CompletableFuture::join).collect(toList())); + }) + .exceptionally( + ex -> { + resultFuture.completeExceptionally(ex); + return null; + }); return resultFuture.join(); } private CompletableFuture get(String host, String path, Class clazz) { - return httpClient.prepareGet(host + path).execute().toCompletableFuture().thenApply(response -> { - try { - if (response.getStatusCode() != 200) { - log.error("Failed to do HTTP get request to {}{} -- code: {}", host, path, response.getStatusCode()); - } - Preconditions.checkArgument(response.getStatusCode() == 200); - return mapper.readValue(response.getResponseBody(), clazz); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + return httpClient + .prepareGet(host + path) + .execute() + .toCompletableFuture() + .thenApply( + response -> { + try { + if (response.getStatusCode() != 200) { + log.error( + "Failed to do HTTP get request to {}{} -- code: {}", + host, + path, + response.getStatusCode()); + } + Preconditions.checkArgument(response.getStatusCode() == 200); + return mapper.readValue(response.getResponseBody(), clazz); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } private CompletableFuture post(String host, String path, byte[] body, Class clazz) { - return httpClient.preparePost(host + path).setBody(body).execute().toCompletableFuture().thenApply(response -> { - try { - if (response.getStatusCode() != 200) { - log.error("Failed to do HTTP post request to {}{} -- code: {}", host, path, response.getStatusCode()); - } - Preconditions.checkArgument(response.getStatusCode() == 200); - return mapper.readValue(response.getResponseBody(), clazz); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + return httpClient + .preparePost(host + path) + .setBody(body) + .execute() + .toCompletableFuture() + .thenApply( + response -> { + try { + if (response.getStatusCode() != 200) { + log.error( + "Failed to do HTTP post request to {}{} -- code: {}", + host, + path, + response.getStatusCode()); + } + Preconditions.checkArgument(response.getStatusCode() == 200); + return mapper.readValue(response.getResponseBody(), clazz); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } @Override @@ -351,13 +417,12 @@ private CompletableFuture[] newArray(int size) { private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter(); - private static final ObjectMapper mapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); static { mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE); } private static final Logger log = LoggerFactory.getLogger(DistributedWorkersEnsemble.class); - } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index 925dd635a..ca5d20b69 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -13,45 +13,24 @@ */ package io.openmessaging.benchmark.worker; -import static io.openmessaging.benchmark.utils.UniformRateLimiter.*; +import static io.openmessaging.benchmark.utils.UniformRateLimiter.uninterruptibleSleepNs; import static java.util.stream.Collectors.toList; -import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.LongAdder; - -import io.openmessaging.benchmark.utils.UniformRateLimiter; -import java.util.stream.IntStream; -import org.HdrHistogram.Recorder; -import org.apache.bookkeeper.stats.Counter; -import org.apache.bookkeeper.stats.NullStatsLogger; -import org.apache.bookkeeper.stats.OpStatsLogger; -import org.apache.bookkeeper.stats.StatsLogger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.base.Preconditions; - import io.netty.util.concurrent.DefaultThreadFactory; import io.openmessaging.benchmark.DriverConfiguration; import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.BenchmarkDriver; +import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo; import io.openmessaging.benchmark.driver.BenchmarkProducer; import io.openmessaging.benchmark.driver.ConsumerCallback; import io.openmessaging.benchmark.utils.RandomGenerator; import io.openmessaging.benchmark.utils.Timer; +import io.openmessaging.benchmark.utils.UniformRateLimiter; import io.openmessaging.benchmark.utils.distributor.KeyDistributor; import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; import io.openmessaging.benchmark.worker.commands.CountersStats; @@ -59,6 +38,28 @@ import io.openmessaging.benchmark.worker.commands.PeriodStats; import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment; import io.openmessaging.benchmark.worker.commands.TopicsInfo; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.IntStream; +import org.HdrHistogram.Recorder; +import org.apache.bookkeeper.stats.Counter; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LocalWorker implements Worker, ConsumerCallback { @@ -69,7 +70,8 @@ public class LocalWorker implements Worker, ConsumerCallback { private volatile UniformRateLimiter rateLimiter = new UniformRateLimiter(1.0); - private final ExecutorService executor = Executors.newCachedThreadPool(new DefaultThreadFactory("local-worker")); + private final ExecutorService executor = + Executors.newCachedThreadPool(new DefaultThreadFactory("local-worker")); // stats @@ -91,17 +93,19 @@ public class LocalWorker implements Worker, ConsumerCallback { private final LongAdder totalMessageSendErrors = new LongAdder(); private final LongAdder totalMessagesReceived = new LongAdder(); - private final static long highestTrackableValue = TimeUnit.SECONDS.toMicros(60); + private static final long highestTrackableValue = TimeUnit.SECONDS.toMicros(60); private final Recorder publishLatencyRecorder = new Recorder(highestTrackableValue, 5); private final Recorder cumulativePublishLatencyRecorder = new Recorder(highestTrackableValue, 5); private final OpStatsLogger publishLatencyStats; private final Recorder publishDelayLatencyRecorder = new Recorder(highestTrackableValue, 5); - private final Recorder cumulativePublishDelayLatencyRecorder = new Recorder(highestTrackableValue, 5); + private final Recorder cumulativePublishDelayLatencyRecorder = + new Recorder(highestTrackableValue, 5); private final OpStatsLogger publishDelayLatencyStats; private final Recorder endToEndLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5); - private final Recorder endToEndCumulativeLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5); + private final Recorder endToEndCumulativeLatencyRecorder = + new Recorder(TimeUnit.HOURS.toMicros(12), 5); private final OpStatsLogger endToEndLatencyStats; private boolean testCompleted = false; @@ -133,14 +137,19 @@ public void initializeDriver(File driverConfigFile) throws IOException { Preconditions.checkArgument(benchmarkDriver == null); testCompleted = false; - DriverConfiguration driverConfiguration = mapper.readValue(driverConfigFile, DriverConfiguration.class); + DriverConfiguration driverConfiguration = + mapper.readValue(driverConfigFile, DriverConfiguration.class); log.info("Driver: {}", writer.writeValueAsString(driverConfiguration)); try { - benchmarkDriver = (BenchmarkDriver) Class.forName(driverConfiguration.driverClass).newInstance(); + benchmarkDriver = + (BenchmarkDriver) Class.forName(driverConfiguration.driverClass).newInstance(); benchmarkDriver.initialize(driverConfigFile, statsLogger); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException | InterruptedException e) { + } catch (InstantiationException + | IllegalAccessException + | ClassNotFoundException + | InterruptedException e) { throw new RuntimeException(e); } } @@ -149,9 +158,11 @@ public void initializeDriver(File driverConfigFile) throws IOException { public List createTopics(TopicsInfo topicsInfo) { Timer timer = new Timer(); - List topicInfos = IntStream.range(0, topicsInfo.numberOfTopics) - .mapToObj(i -> new TopicInfo(generateTopicName(i), topicsInfo.numberOfPartitionsPerTopic)) - .collect(toList()); + List topicInfos = + IntStream.range(0, topicsInfo.numberOfTopics) + .mapToObj( + i -> new TopicInfo(generateTopicName(i), topicsInfo.numberOfPartitionsPerTopic)) + .collect(toList()); benchmarkDriver.createTopics(topicInfos).join(); @@ -162,15 +173,16 @@ public List createTopics(TopicsInfo topicsInfo) { } private String generateTopicName(int i) { - return String.format("%s-%07d-%s", benchmarkDriver.getTopicNamePrefix(), i, RandomGenerator.getRandomString()); + return String.format( + "%s-%07d-%s", benchmarkDriver.getTopicNamePrefix(), i, RandomGenerator.getRandomString()); } @Override public void createProducers(List topics) { Timer timer = new Timer(); - List> futures = topics.stream() - .map(topic -> benchmarkDriver.createProducer(topic)).collect(toList()); + List> futures = + topics.stream().map(topic -> benchmarkDriver.createProducer(topic)).collect(toList()); futures.forEach(f -> producers.add(f.join())); log.info("Created {} producers in {} ms", producers.size(), timer.elapsedMillis()); @@ -180,8 +192,10 @@ public void createProducers(List topics) { public void createConsumers(ConsumerAssignment consumerAssignment) { Timer timer = new Timer(); - List> futures = consumerAssignment.topicsSubscriptions.stream() - .map(ts -> benchmarkDriver.createConsumer(ts.topic, ts.subscription, this)).collect(toList()); + List> futures = + consumerAssignment.topicsSubscriptions.stream() + .map(ts -> benchmarkDriver.createConsumer(ts.topic, ts.subscription, this)) + .collect(toList()); futures.forEach(f -> consumers.add(f.join())); log.info("Created {} consumers in {} ms", consumers.size(), timer.elapsedMillis()); @@ -197,71 +211,97 @@ public void startLoad(ProducerWorkAssignment producerWorkAssignment) { int processorIdx = 0; for (BenchmarkProducer p : producers) { - processorAssignment.computeIfAbsent(processorIdx, x -> new ArrayList()).add(p); + processorAssignment + .computeIfAbsent(processorIdx, x -> new ArrayList()) + .add(p); processorIdx = (processorIdx + 1) % processors; } - processorAssignment.values().forEach(producers -> submitProducersToExecutor(producers, - KeyDistributor.build(producerWorkAssignment.keyDistributorType), producerWorkAssignment.payloadData)); + processorAssignment + .values() + .forEach( + producers -> + submitProducersToExecutor( + producers, + KeyDistributor.build(producerWorkAssignment.keyDistributorType), + producerWorkAssignment.payloadData)); } @Override public void probeProducers() throws IOException { - producers.forEach(producer -> producer.sendAsync(Optional.of("key"), new byte[10]) - .thenRun(() -> totalMessagesSent.increment())); + producers.forEach( + producer -> + producer + .sendAsync(Optional.of("key"), new byte[10]) + .thenRun(() -> totalMessagesSent.increment())); } - private void submitProducersToExecutor(List producers, KeyDistributor keyDistributor, List payloads) { - executor.submit(() -> { - int payloadCount = payloads.size(); - ThreadLocalRandom r = ThreadLocalRandom.current(); - byte[] firstPayload = payloads.get(0); - - try { - while (!testCompleted) { - producers.forEach(producer -> { - byte[] payloadData = payloadCount == 0 ? firstPayload : payloads.get(r.nextInt(payloadCount)); - final long intendedSendTime = rateLimiter.acquire(); - uninterruptibleSleepNs(intendedSendTime); - final long sendTime = System.nanoTime(); - producer.sendAsync(Optional.ofNullable(keyDistributor.next()), payloadData) - .thenRun(() -> { - messagesSent.increment(); - totalMessagesSent.increment(); - messagesSentCounter.inc(); - bytesSent.add(payloadData.length); - bytesSentCounter.add(payloadData.length); - - final long latencyMicros = Math.min(highestTrackableValue, - TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sendTime)); - publishLatencyRecorder.recordValue(latencyMicros); - cumulativePublishLatencyRecorder.recordValue(latencyMicros); - publishLatencyStats.registerSuccessfulEvent(latencyMicros, TimeUnit.MICROSECONDS); - - final long sendDelayMicros = Math.min(highestTrackableValue, - TimeUnit.NANOSECONDS.toMicros(sendTime - intendedSendTime)); - publishDelayLatencyRecorder.recordValue(sendDelayMicros); - cumulativePublishDelayLatencyRecorder.recordValue(sendDelayMicros); - publishDelayLatencyStats.registerSuccessfulEvent(sendDelayMicros, TimeUnit.MICROSECONDS); - }).exceptionally(ex -> { - messageSendErrors.increment(); - messageSendErrorCounter.inc(); - totalMessageSendErrors.increment(); - log.warn("Write error on message", ex); - return null; - }); - }); - } - } catch (Throwable t) { - log.error("Got error", t); - } - }); + @SuppressWarnings("checkstyle:LineLength") + private void submitProducersToExecutor( + List producers, KeyDistributor keyDistributor, List payloads) { + executor.submit( + () -> { + int payloadCount = payloads.size(); + ThreadLocalRandom r = ThreadLocalRandom.current(); + byte[] firstPayload = payloads.get(0); + + try { + while (!testCompleted) { + producers.forEach( + producer -> { + byte[] payloadData = + payloadCount == 0 ? firstPayload : payloads.get(r.nextInt(payloadCount)); + final long intendedSendTime = rateLimiter.acquire(); + uninterruptibleSleepNs(intendedSendTime); + final long sendTime = System.nanoTime(); + producer + .sendAsync(Optional.ofNullable(keyDistributor.next()), payloadData) + .thenRun( + () -> { + messagesSent.increment(); + totalMessagesSent.increment(); + messagesSentCounter.inc(); + bytesSent.add(payloadData.length); + bytesSentCounter.add(payloadData.length); + + final long latencyMicros = + Math.min( + highestTrackableValue, + TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sendTime)); + publishLatencyRecorder.recordValue(latencyMicros); + cumulativePublishLatencyRecorder.recordValue(latencyMicros); + publishLatencyStats.registerSuccessfulEvent( + latencyMicros, TimeUnit.MICROSECONDS); + + final long sendDelayMicros = + Math.min( + highestTrackableValue, + TimeUnit.NANOSECONDS.toMicros(sendTime - intendedSendTime)); + publishDelayLatencyRecorder.recordValue(sendDelayMicros); + cumulativePublishDelayLatencyRecorder.recordValue(sendDelayMicros); + publishDelayLatencyStats.registerSuccessfulEvent( + sendDelayMicros, TimeUnit.MICROSECONDS); + }) + .exceptionally( + ex -> { + messageSendErrors.increment(); + messageSendErrorCounter.inc(); + totalMessageSendErrors.increment(); + log.warn("Write error on message", ex); + return null; + }); + }); + } + } catch (Throwable t) { + log.error("Got error", t); + } + }); } @Override public void adjustPublishRate(double publishRate) { - if(publishRate < 1.0) { + if (publishRate < 1.0) { rateLimiter = new UniformRateLimiter(1.0); return; } @@ -412,8 +452,9 @@ public void close() throws Exception { private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter(); - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); static { mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java index fa9dde4d9..5333f516e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java @@ -13,9 +13,6 @@ */ package io.openmessaging.benchmark.worker; -import java.io.File; -import java.io.IOException; -import java.util.List; import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; import io.openmessaging.benchmark.worker.commands.CountersStats; @@ -23,6 +20,9 @@ import io.openmessaging.benchmark.worker.commands.PeriodStats; import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment; import io.openmessaging.benchmark.worker.commands.TopicsInfo; +import java.io.File; +import java.io.IOException; +import java.util.List; public interface Worker extends AutoCloseable { diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java index 951fab4e9..05fad840e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java @@ -13,19 +13,11 @@ */ package io.openmessaging.benchmark.worker; -import java.io.File; -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.bookkeeper.stats.StatsLogger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.io.Files; - import io.javalin.Context; import io.javalin.Javalin; import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; @@ -33,6 +25,12 @@ import io.openmessaging.benchmark.worker.commands.PeriodStats; import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment; import io.openmessaging.benchmark.worker.commands.TopicsInfo; +import java.io.File; +import java.nio.ByteBuffer; +import java.util.List; +import org.apache.bookkeeper.stats.StatsLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @SuppressWarnings("unchecked") public class WorkerHandler { @@ -87,7 +85,8 @@ private void handleProbeProducers(Context ctx) throws Exception { private void handleCreateConsumers(Context ctx) throws Exception { ConsumerAssignment consumerAssignment = mapper.readValue(ctx.body(), ConsumerAssignment.class); - log.info("Received create consumers request for topics: {}", consumerAssignment.topicsSubscriptions); + log.info( + "Received create consumers request for topics: {}", consumerAssignment.topicsSubscriptions); localWorker.createConsumers(consumerAssignment); } @@ -100,9 +99,12 @@ private void handleResumeConsumers(Context ctx) throws Exception { } private void handleStartLoad(Context ctx) throws Exception { - ProducerWorkAssignment producerWorkAssignment = mapper.readValue(ctx.body(), ProducerWorkAssignment.class); + ProducerWorkAssignment producerWorkAssignment = + mapper.readValue(ctx.body(), ProducerWorkAssignment.class); - log.info("Start load publish-rate: {} msg/s -- payload-size: {}", producerWorkAssignment.publishRate, + log.info( + "Start load publish-rate: {} msg/s -- payload-size: {}", + producerWorkAssignment.publishRate, producerWorkAssignment.payloadData.get(0).length); localWorker.startLoad(producerWorkAssignment); @@ -186,13 +188,12 @@ private void handleResetStats(Context ctx) throws Exception { private static final Logger log = LoggerFactory.getLogger(WorkerHandler.class); - private static final ObjectMapper mapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); static { mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE); } private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter(); - } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java index 3ae9c97e7..9d97642ec 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ConsumerAssignment.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.worker.commands; + import java.util.ArrayList; import java.util.List; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java index 92a12ef7b..e489d9107 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java @@ -13,23 +13,21 @@ */ package io.openmessaging.benchmark.worker.commands; -import java.util.concurrent.TimeUnit; - -import org.HdrHistogram.Histogram; import com.fasterxml.jackson.annotation.JsonIgnore; +import java.util.concurrent.TimeUnit; +import org.HdrHistogram.Histogram; public class CumulativeLatencies { - @JsonIgnore - public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); + @JsonIgnore public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); public byte[] publishLatencyBytes; @JsonIgnore public Histogram publishDelayLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); + public byte[] publishDelayLatencyBytes; - @JsonIgnore - public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5); + @JsonIgnore public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5); public byte[] endToEndLatencyBytes; } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java index 2dbe57d72..ec99f75dd 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java @@ -13,11 +13,10 @@ */ package io.openmessaging.benchmark.worker.commands; -import java.util.concurrent.TimeUnit; - -import org.HdrHistogram.Histogram; import com.fasterxml.jackson.annotation.JsonIgnore; +import java.util.concurrent.TimeUnit; +import org.HdrHistogram.Histogram; public class PeriodStats { public long messagesSent = 0; @@ -31,16 +30,14 @@ public class PeriodStats { public long totalMessageSendErrors = 0; public long totalMessagesReceived = 0; - @JsonIgnore - public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); + @JsonIgnore public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); public byte[] publishLatencyBytes; @JsonIgnore public Histogram publishDelayLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); - public byte[] publishDelayLatencyBytes; + public byte[] publishDelayLatencyBytes; - @JsonIgnore - public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5); + @JsonIgnore public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5); public byte[] endToEndLatencyBytes; } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java index 53c7e8869..50dfe0704 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java @@ -13,14 +13,14 @@ */ package io.openmessaging.benchmark.worker.commands; -import io.openmessaging.benchmark.utils.distributor.KeyDistributorType; +import io.openmessaging.benchmark.utils.distributor.KeyDistributorType; import java.util.List; public class ProducerWorkAssignment { - + public List payloadData; - + public double publishRate; public KeyDistributorType keyDistributorType; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java index f21ce6745..bd9bf9989 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicSubscription.java @@ -17,8 +17,7 @@ public class TopicSubscription { public String topic; public String subscription; - public TopicSubscription() { - } + public TopicSubscription() {} public TopicSubscription(String topic, String subscription) { this.topic = topic; @@ -27,9 +26,13 @@ public TopicSubscription(String topic, String subscription) { @Override public String toString() { - return "TopicSubscription{" + - "topic='" + topic + '\'' + - ", subscription='" + subscription + '\'' + - '}'; + return "TopicSubscription{" + + "topic='" + + topic + + '\'' + + ", subscription='" + + subscription + + '\'' + + '}'; } } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java index b277a8f5f..90545c725 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/TopicsInfo.java @@ -17,8 +17,7 @@ public class TopicsInfo { public int numberOfTopics; public int numberOfPartitionsPerTopic; - public TopicsInfo() { - } + public TopicsInfo() {} public TopicsInfo(int numberOfTopics, int numberOfPartitionsPerTopic) { this.numberOfTopics = numberOfTopics; diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java index b3e7c0a53..a7472df95 100644 --- a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/ListPartitionTest.java @@ -15,6 +15,7 @@ import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; + import java.util.List; import org.junit.jupiter.api.Test; @@ -24,11 +25,13 @@ class ListPartitionTest { void partitionList() { List list = asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); List> lists = ListPartition.partitionList(list, 3); - assertThat(lists).satisfies(s -> { - assertThat(s).hasSize(3); - assertThat(s.get(0)).isEqualTo(asList(1, 4, 7, 10)); - assertThat(s.get(1)).isEqualTo(asList(2, 5, 8)); - assertThat(s.get(2)).isEqualTo(asList(3, 6, 9)); - }); + assertThat(lists) + .satisfies( + s -> { + assertThat(s).hasSize(3); + assertThat(s.get(0)).isEqualTo(asList(1, 4, 7, 10)); + assertThat(s.get(1)).isEqualTo(asList(2, 5, 8)); + assertThat(s.get(2)).isEqualTo(asList(3, 6, 9)); + }); } -} \ No newline at end of file +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java index dfe52031a..be1b9a532 100644 --- a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java @@ -14,6 +14,7 @@ package io.openmessaging.benchmark.utils; import static org.assertj.core.api.Assertions.assertThat; + import org.junit.jupiter.api.Test; class PaddingDecimalFormatTest { @@ -25,5 +26,4 @@ void format() { assertThat(format.format(1000L)).isEqualTo(" 1000.0"); assertThat(format.format(10000000L)).isEqualTo("10000000.0"); } - -} \ No newline at end of file +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java index 5d52fbe01..87004e332 100644 --- a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/TimerTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; + import java.util.function.Supplier; import org.junit.jupiter.api.Test; @@ -38,4 +39,4 @@ void elapsedSeconds() { Timer timer = new Timer(mockClock); assertThat(timer.elapsedSeconds()).isEqualTo(2.0d); } -} \ No newline at end of file +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java index b9f200846..a9b90462e 100644 --- a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/UniformRateLimiterTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; + import java.util.function.Supplier; import org.junit.jupiter.api.Test; @@ -55,9 +56,11 @@ void uninterruptibleSleepNs() { @Test void cinitExceptions() { - assertThatCode(() -> new UniformRateLimiter(Double.NaN)).isInstanceOf(IllegalArgumentException.class); - assertThatCode(() -> new UniformRateLimiter(1.0d / 0.0d)).isInstanceOf(IllegalArgumentException.class); + assertThatCode(() -> new UniformRateLimiter(Double.NaN)) + .isInstanceOf(IllegalArgumentException.class); + assertThatCode(() -> new UniformRateLimiter(1.0d / 0.0d)) + .isInstanceOf(IllegalArgumentException.class); assertThatCode(() -> new UniformRateLimiter(-0.1)).isInstanceOf(IllegalArgumentException.class); assertThatCode(() -> new UniformRateLimiter(0.0)).isInstanceOf(IllegalArgumentException.class); } -} \ No newline at end of file +} diff --git a/deployment/kubernetes/helm/README.md b/deployment/kubernetes/helm/README.md index ac43ac652..f8ef3125a 100644 --- a/deployment/kubernetes/helm/README.md +++ b/deployment/kubernetes/helm/README.md @@ -1,11 +1,10 @@ - Users can deploy the helm chart: ```bash $ helm install ./benchmark --name benchmark ``` -After the chart has started, users can exec into the pod name "benchmark-driver" and run the benchmark from there. +After the chart has started, users can exec into the pod name "benchmark-driver" and run the benchmark from there. For example, once inside the "benchmark-driver" pod, users can execute: diff --git a/docker/README.md b/docker/README.md index 1cefa026d..a58ef527c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,9 +1,12 @@ # OpenMessaging Benchmark Framework Docker -You can use either of the Dockerfiles - `./docker/Dockerfile` or `./docker/Dockerfile.build` based on your needs. + +You can use either of the Dockerfiles - `./docker/Dockerfile` or `./docker/Dockerfile.build` based on your needs. ### `Dockerfile` + Uses `openjdk-8` and takes `BENCHMARK_TARBALL` as an argument. While using this Dockerfile, you will need to build the project locally **first**. + ``` #> mvn build #> export BENCHMARK_TARBALL=package/target/openmessaging-benchmark--SNAPSHOT-bin.tar.gz @@ -11,8 +14,11 @@ While using this Dockerfile, you will need to build the project locally **first* ``` ### `Dockerfile.build` + Uses the latest version of `maven` in order to build the project, and then use `openjdk-8` as runtime. This Dockerfile has no dependency (you do not need Mavent to be installed locally). + ``` #> docker build . -f docker/Dockerfile.build -``` \ No newline at end of file +``` + diff --git a/docker/pom.xml b/docker/pom.xml index bf5188238..6f9925976 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -1,3 +1,4 @@ + + 4.0.0 - messaging-benchmark io.openmessaging.benchmark + messaging-benchmark 0.0.1-SNAPSHOT - 4.0.0 messaging-benchmark-docker + pom Open Messaging Benchmark Docker Image - pom 1.3.7 @@ -52,6 +53,14 @@ com.spotify dockerfile-maven-plugin ${dockerfile-maven.version} + + ${docker.organization}/openmessaging-benchmark + false + ${project.version} + + target/package-${project.version}-bin.tar.gz + + default @@ -80,14 +89,6 @@ - - ${docker.organization}/openmessaging-benchmark - false - ${project.version} - - target/package-${project.version}-bin.tar.gz - - org.apache.maven.plugins diff --git a/driver-api/pom.xml b/driver-api/pom.xml index c26dd24d3..cbf66b392 100644 --- a/driver-api/pom.xml +++ b/driver-api/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - driver-api + driver-api - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-base - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${bookkeeper.version} - - - org.projectlombok - lombok - - + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-base + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + + + org.apache.bookkeeper.stats + bookkeeper-stats-api + ${bookkeeper.version} + + + org.projectlombok + lombok + + diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java index 1788dc6a5..a8e745e88 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkConsumer.java @@ -13,6 +13,4 @@ */ package io.openmessaging.benchmark.driver; -public interface BenchmarkConsumer extends AutoCloseable { - -} +public interface BenchmarkConsumer extends AutoCloseable {} diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java index 920e3c41e..32f18f082 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver; + import java.io.File; import java.io.IOException; import java.util.List; @@ -20,63 +21,73 @@ import lombok.Value; import org.apache.bookkeeper.stats.StatsLogger; -/** - * Base driver interface - */ +/** Base driver interface. */ public interface BenchmarkDriver extends AutoCloseable { /** - * Driver implementation can use this method to initialize the client libraries, with the provided configuration - * file. - *

- * The format of the configuration file is specific to the driver implementation. - * + * Driver implementation can use this method to initialize the client libraries, with the provided + * configuration file. + * + *

The format of the configuration file is specific to the driver implementation. + * * @param configurationFile * @param statsLogger stats logger to collect stats from benchmark driver * @throws IOException */ - void initialize(File configurationFile, StatsLogger statsLogger) throws IOException, InterruptedException; + void initialize(File configurationFile, StatsLogger statsLogger) + throws IOException, InterruptedException; /** - * Get a driver specific prefix to be used in creating multiple topic names + * Get a driver specific prefix to be used in creating multiple topic names. + * + * @return the topic name prefix */ String getTopicNamePrefix(); /** - * Create a new topic with a given number of partitions + * Create a new topic with a given number of partitions. + * + * @param topic + * @param partitions + * @return a future the completes when the topic is created */ CompletableFuture createTopic(String topic, int partitions); /** - * Create a list of new topics with the given number of partitions + * Create a list of new topics with the given number of partitions. + * + * @param topicInfos + * @return a future the completes when the topics are created */ default CompletableFuture createTopics(List topicInfos) { @SuppressWarnings("unchecked") - CompletableFuture[] futures = topicInfos.stream() - .map(topicInfo -> createTopic(topicInfo.getTopic(), topicInfo.getPartitions())) - .toArray(CompletableFuture[]::new); + CompletableFuture[] futures = + topicInfos.stream() + .map(topicInfo -> createTopic(topicInfo.getTopic(), topicInfo.getPartitions())) + .toArray(CompletableFuture[]::new); return CompletableFuture.allOf(futures); } /** - * Create a producer for a given topic + * Create a producer for a given topic. + * + * @param topic + * @return a producer future */ CompletableFuture createProducer(String topic); /** * Create a benchmark consumer relative to one particular topic and subscription. - * - * It is responsibility of the driver implementation to invoke the consumerCallback each time a message - * is received. - * + * + *

It is responsibility of the driver implementation to invoke the consumerCallback + * each time a message is received. + * * @param topic * @param subscriptionName * @param consumerCallback - * @return + * @return a consumer future */ CompletableFuture createConsumer( - String topic, - String subscriptionName, - ConsumerCallback consumerCallback); + String topic, String subscriptionName, ConsumerCallback consumerCallback); @Value class TopicInfo { diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java index da9bf06ac..1662cf1c6 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkProducer.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver; + import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -21,12 +22,9 @@ public interface BenchmarkProducer extends AutoCloseable { /** * Publish a message and return a callback to track the completion of the operation. * - * @param key - * the key associated with this message - * @param payload - * the message payload + * @param key the key associated with this message + * @param payload the message payload * @return a future that will be triggered when the message is successfully published */ CompletableFuture sendAsync(Optional key, byte[] payload); - } diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java index 88817009f..b68ecf451 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ConsumerCallback.java @@ -13,29 +13,24 @@ */ package io.openmessaging.benchmark.driver; + import java.nio.ByteBuffer; -/** - * Callback that the driver implementation calls when a message is received - */ +/** Callback that the driver implementation calls when a message is received. */ public interface ConsumerCallback { /** - * Driver should invoke this method (or the ByteBuffer variant) once for each message received - * - * @param payload - * the received message payload - * @param publishTimestamp - * the publish timestamp in milliseconds + * Driver should invoke this method (or the ByteBuffer variant) once for each message received. + * + * @param payload the received message payload + * @param publishTimestamp the publish timestamp in milliseconds */ void messageReceived(byte[] payload, long publishTimestamp); /** - * Driver should invoke this method (or the byte[] variant) once for each message received + * Driver should invoke this method (or the byte[] variant) once for each message received. * - * @param payload - * the received message payload - * @param publishTimestamp - * the publish timestamp in milliseconds + * @param payload the received message payload + * @param publishTimestamp the publish timestamp in milliseconds */ void messageReceived(ByteBuffer payload, long publishTimestamp); } diff --git a/driver-artemis/pom.xml b/driver-artemis/pom.xml index 9c1f9989a..fe8eed01d 100644 --- a/driver-artemis/pom.xml +++ b/driver-artemis/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - driver-artemis + driver-artemis - - - - ${project.groupId} - driver-api - ${project.version} - - - org.apache.activemq - artemis-core-client - 2.23.1 - - + + + ${project.groupId} + driver-api + ${project.version} + + + org.apache.activemq + artemis-core-client + 2.23.1 + + diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java index c714e71fc..e0b988219 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkConsumer.java @@ -13,6 +13,9 @@ */ package io.openmessaging.benchmark.driver.artemis; + +import io.openmessaging.benchmark.driver.BenchmarkConsumer; +import io.openmessaging.benchmark.driver.ConsumerCallback; import org.apache.activemq.artemis.api.core.ActiveMQException; import org.apache.activemq.artemis.api.core.RoutingType; import org.apache.activemq.artemis.api.core.SimpleString; @@ -22,31 +25,36 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.openmessaging.benchmark.driver.BenchmarkConsumer; -import io.openmessaging.benchmark.driver.ConsumerCallback; - public class ArtemisBenchmarkConsumer implements BenchmarkConsumer { private final ClientSession session; private final ClientConsumer consumer; - public ArtemisBenchmarkConsumer(String topic, String queueName, ClientSessionFactory sessionFactory, ConsumerCallback callback) + public ArtemisBenchmarkConsumer( + String topic, + String queueName, + ClientSessionFactory sessionFactory, + ConsumerCallback callback) throws ActiveMQException { session = sessionFactory.createSession(); - session.createQueue(SimpleString.toSimpleString(topic), RoutingType.MULTICAST, - SimpleString.toSimpleString(queueName), true /* durable */); + session.createQueue( + SimpleString.toSimpleString(topic), + RoutingType.MULTICAST, + SimpleString.toSimpleString(queueName), + true /* durable */); consumer = session.createConsumer(queueName); - consumer.setMessageHandler(message -> { - byte[] payload = new byte[message.getBodyBuffer().readableBytes()]; - message.getBodyBuffer().readBytes(payload); - callback.messageReceived(payload, message.getTimestamp()); - try { - message.acknowledge(); - } catch (ActiveMQException e) { - log.warn("Failed to acknowledge message", e); - } - }); - + consumer.setMessageHandler( + message -> { + byte[] payload = new byte[message.getBodyBuffer().readableBytes()]; + message.getBodyBuffer().readBytes(payload); + callback.messageReceived(payload, message.getTimestamp()); + try { + message.acknowledge(); + } catch (ActiveMQException e) { + log.warn("Failed to acknowledge message", e); + } + }); + session.start(); } diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java index ae49824c1..f1a966ca2 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java @@ -13,11 +13,19 @@ */ package io.openmessaging.benchmark.driver.artemis; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.openmessaging.benchmark.driver.BenchmarkConsumer; +import io.openmessaging.benchmark.driver.BenchmarkDriver; +import io.openmessaging.benchmark.driver.BenchmarkProducer; +import io.openmessaging.benchmark.driver.ConsumerCallback; import java.io.File; import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ForkJoinPool; - import org.apache.activemq.artemis.api.core.ActiveMQException; import org.apache.activemq.artemis.api.core.RoutingType; import org.apache.activemq.artemis.api.core.SimpleString; @@ -29,16 +37,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - -import io.openmessaging.benchmark.driver.BenchmarkConsumer; -import io.openmessaging.benchmark.driver.BenchmarkDriver; -import io.openmessaging.benchmark.driver.BenchmarkProducer; -import io.openmessaging.benchmark.driver.ConsumerCallback; - public class ArtemisBenchmarkDriver implements BenchmarkDriver { private ArtemisConfig config; @@ -52,7 +50,7 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I try { ServerLocator serverLocator = ActiveMQClient.createServerLocator(config.brokerAddress); serverLocator.setConfirmationWindowSize(1000); - + sessionFactory = serverLocator.createSessionFactory(); session = sessionFactory.createSession(); } catch (Exception e) { @@ -69,18 +67,22 @@ public String getTopicNamePrefix() { public CompletableFuture createTopic(String topic, int partitions) { CompletableFuture future = new CompletableFuture<>(); if (partitions != 1) { - future.completeExceptionally(new IllegalArgumentException("Partitions are not supported in Artemis")); + future.completeExceptionally( + new IllegalArgumentException("Partitions are not supported in Artemis")); return future; } - ForkJoinPool.commonPool().submit(() -> { - try { - session.createAddress(SimpleString.toSimpleString(topic), RoutingType.MULTICAST, true); - future.complete(null); - } catch (ActiveMQException e) { - future.completeExceptionally(e); - } - }); + ForkJoinPool.commonPool() + .submit( + () -> { + try { + session.createAddress( + SimpleString.toSimpleString(topic), RoutingType.MULTICAST, true); + future.complete(null); + } catch (ActiveMQException e) { + future.completeExceptionally(e); + } + }); return future; } @@ -97,19 +99,22 @@ public CompletableFuture createProducer(String topic) { } @Override - public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { CompletableFuture future = new CompletableFuture<>(); - ForkJoinPool.commonPool().submit(() -> { - try { - String queueName = topic + "-" + subscriptionName; - BenchmarkConsumer consumer = new ArtemisBenchmarkConsumer(topic, queueName, sessionFactory, - consumerCallback); - future.complete(consumer); - } catch (ActiveMQException e) { - future.completeExceptionally(e); - } - }); + ForkJoinPool.commonPool() + .submit( + () -> { + try { + String queueName = topic + "-" + subscriptionName; + BenchmarkConsumer consumer = + new ArtemisBenchmarkConsumer( + topic, queueName, sessionFactory, consumerCallback); + future.complete(consumer); + } catch (ActiveMQException e) { + future.completeExceptionally(e); + } + }); return future; } @@ -129,8 +134,9 @@ public void close() throws Exception { log.info("ActiveMQ Artemis benchmark driver successfully shut down"); } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static ArtemisConfig readConfig(File configurationFile) throws IOException { return mapper.readValue(configurationFile, ArtemisConfig.class); diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java index baaa8cf7c..622d37ea8 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java @@ -13,23 +13,23 @@ */ package io.openmessaging.benchmark.driver.artemis; + +import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.Optional; import java.util.concurrent.CompletableFuture; - import org.apache.activemq.artemis.api.core.ActiveMQException; import org.apache.activemq.artemis.api.core.client.ClientMessage; import org.apache.activemq.artemis.api.core.client.ClientProducer; import org.apache.activemq.artemis.api.core.client.ClientSession; import org.apache.activemq.artemis.api.core.client.ClientSessionFactory; -import io.openmessaging.benchmark.driver.BenchmarkProducer; - public class ArtemisBenchmarkProducer implements BenchmarkProducer { private final ClientSession session; private final ClientProducer producer; - public ArtemisBenchmarkProducer(String address, ClientSessionFactory sessionFactory) throws ActiveMQException { + public ArtemisBenchmarkProducer(String address, ClientSessionFactory sessionFactory) + throws ActiveMQException { session = sessionFactory.createSession(); producer = session.createProducer(address); session.start(); @@ -43,20 +43,21 @@ public void close() throws Exception { @Override public CompletableFuture sendAsync(Optional key, byte[] payload) { - ClientMessage msg = session.createMessage(true /* durable */ ); + ClientMessage msg = session.createMessage(true /* durable */); msg.setTimestamp(System.currentTimeMillis()); msg.getBodyBuffer().writeBytes(payload); CompletableFuture future = new CompletableFuture<>(); try { - producer.send(msg, message -> { - future.complete(null); - }); + producer.send( + msg, + message -> { + future.complete(null); + }); } catch (ActiveMQException e) { future.completeExceptionally(e); } return future; } - } diff --git a/driver-bookkeeper/pom.xml b/driver-bookkeeper/pom.xml index c24790138..3bbea8691 100644 --- a/driver-bookkeeper/pom.xml +++ b/driver-bookkeeper/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - driver-bookkeeper - - - ${project.groupId} - driver-api - ${project.version} - - - org.apache.distributedlog - distributedlog-core-shaded - 4.14.4 - - - io.netty - netty-all - - + driver-bookkeeper + + + ${project.groupId} + driver-api + ${project.version} + + + io.netty + netty-all + + + org.apache.distributedlog + distributedlog-core-shaded + 4.14.4 + + diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java index 04b660c50..db8c83a0b 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/Config.java @@ -18,5 +18,4 @@ public class Config { public String dlogUri; public String dlogConf; - } diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java index 88657e6ee..95414ef84 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkConsumer.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.bookkeeper; + import dlshade.com.google.common.util.concurrent.ThreadFactoryBuilder; import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.ConsumerCallback; @@ -44,64 +45,74 @@ private static boolean backoff(long backoffTime, TimeUnit timeUnit) { return true; } catch (InterruptedException e) { Thread.currentThread().interrupt(); - log.warn("Interrupted at backoff {} ms", - timeUnit.toMillis(backoffTime), e); + log.warn("Interrupted at backoff {} ms", timeUnit.toMillis(backoffTime), e); return false; } } - public DlogBenchmarkConsumer(DistributedLogManager dlm, - ConsumerCallback callback) { + @SuppressWarnings("checkstyle:LineLength") + public DlogBenchmarkConsumer(DistributedLogManager dlm, ConsumerCallback callback) { this.dlm = dlm; - this.executor = Executors.newSingleThreadExecutor( - new ThreadFactoryBuilder().setNameFormat("dlog-benchmark-reader-thread-%d").build()); + this.executor = + Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder().setNameFormat("dlog-benchmark-reader-thread-%d").build()); - this.readerTask = executor.submit(() -> { - LogReader reader = null; - DLSN lastDLSN = DLSN.InitialDLSN; - LogRecordWithDLSN record; + this.readerTask = + executor.submit( + () -> { + LogReader reader = null; + DLSN lastDLSN = DLSN.InitialDLSN; + LogRecordWithDLSN record; - while (!closing) { - if (null == reader) { - try { - reader = dlm.openLogReader(lastDLSN); - log.info("Successfully open log reader for stream {} at {}", - dlm.getStreamName(), lastDLSN); - } catch (IOException e) { - log.error("Failed to open reader of stream {} at {}", - dlm.getStreamName(), lastDLSN, e); - if (backoff(10, TimeUnit.SECONDS)) { - continue; - } else { - break; - } - } - } + while (!closing) { + if (null == reader) { + try { + reader = dlm.openLogReader(lastDLSN); + log.info( + "Successfully open log reader for stream {} at {}", + dlm.getStreamName(), + lastDLSN); + } catch (IOException e) { + log.error( + "Failed to open reader of stream {} at {}", + dlm.getStreamName(), + lastDLSN, + e); + if (backoff(10, TimeUnit.SECONDS)) { + continue; + } else { + break; + } + } + } - try { - record = reader.readNext(false); - if (null == record) { - try { - Thread.sleep(1); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - continue; - } + try { + record = reader.readNext(false); + if (null == record) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + continue; + } - callback.messageReceived(record.getPayload(), record.getTransactionId()); + callback.messageReceived(record.getPayload(), record.getTransactionId()); - lastDLSN = record.getDlsn(); - } catch (IOException e) { - log.info("Encountered error on reading records from reading stream {}, last record = {}", - dlm.getStreamName(), lastDLSN, e); - Utils.closeQuietly(reader); - reader = null; - } - } + lastDLSN = record.getDlsn(); + } catch (IOException e) { + log.info( + "Encountered error on reading records from reading stream {}, last record = {}", + dlm.getStreamName(), + lastDLSN, + e); + Utils.closeQuietly(reader); + reader = null; + } + } - Utils.closeQuietly(reader); - }); + Utils.closeQuietly(reader); + }); } @Override @@ -113,5 +124,4 @@ public void close() throws Exception { dlm.close(); } } - } diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java index b1ea6071c..fd60a4cdf 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkDriver.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.bookkeeper; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -21,7 +22,6 @@ import io.openmessaging.benchmark.driver.BenchmarkDriver; import io.openmessaging.benchmark.driver.BenchmarkProducer; import io.openmessaging.benchmark.driver.ConsumerCallback; - import io.openmessaging.benchmark.driver.bookkeeper.stats.StatsLoggerAdaptor; import java.io.File; import java.io.IOException; @@ -38,14 +38,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Benchmark driver testing distributedlog. - */ +/** Benchmark driver testing distributedlog. */ public class DlogBenchmarkDriver implements BenchmarkDriver { private static final Logger log = LoggerFactory.getLogger(DlogBenchmarkProducer.class); - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private Config config; private Namespace namespace; @@ -66,13 +65,10 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I URI dlogUri = URI.create(config.dlogUri); dlshade.org.apache.bookkeeper.stats.StatsLogger dlStatsLogger = - new CachingStatsLogger(new StatsLoggerAdaptor(statsLogger.scope("dlog"))); + new CachingStatsLogger(new StatsLoggerAdaptor(statsLogger.scope("dlog"))); - namespace = NamespaceBuilder.newBuilder() - .conf(conf) - .uri(dlogUri) - .statsLogger(dlStatsLogger) - .build(); + namespace = + NamespaceBuilder.newBuilder().conf(conf).uri(dlogUri).statsLogger(dlStatsLogger).build(); log.info("Initialized distributedlog namespace at {}", dlogUri); } @@ -88,53 +84,53 @@ private static String getFullyQualifiedPartitionedStreamName(String topic, int p @Override public CompletableFuture createTopic(String topic, int partitions) { - return CompletableFuture.runAsync(() -> { - try { - namespace.createLog(topic); - if (partitions > 1) { - for (int i = 0; i < partitions; i++) { - namespace.createLog(getFullyQualifiedPartitionedStreamName(topic, i)); + return CompletableFuture.runAsync( + () -> { + try { + namespace.createLog(topic); + if (partitions > 1) { + for (int i = 0; i < partitions; i++) { + namespace.createLog(getFullyQualifiedPartitionedStreamName(topic, i)); + } + } + log.info("Successfully create topic {} with {} partitions", topic, partitions); + } catch (IOException ioe) { + log.error("Failed to create topic {} with {} partitions", topic, partitions, ioe); + throw new RuntimeException(ioe); } - } - log.info("Successfully create topic {} with {} partitions", topic, partitions); - } catch (IOException ioe) { - log.error("Failed to create topic {} with {} partitions", - topic, partitions, ioe); - throw new RuntimeException(ioe); - } - }); + }); } @Override public CompletableFuture createProducer(String topic) { - return CompletableFuture.supplyAsync(() -> { - try { - DistributedLogManager dlm = namespace.openLog(topic); - log.info("Open stream {} for producer", topic); - return dlm; - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - }) - .thenCompose(dlm -> dlm.openAsyncLogWriter()) - .thenApply(writer -> new DlogBenchmarkProducer(writer)); + return CompletableFuture.supplyAsync( + () -> { + try { + DistributedLogManager dlm = namespace.openLog(topic); + log.info("Open stream {} for producer", topic); + return dlm; + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + }) + .thenCompose(dlm -> dlm.openAsyncLogWriter()) + .thenApply(writer -> new DlogBenchmarkProducer(writer)); } @Override public CompletableFuture createConsumer( - String topic, - String subscriptionName, - ConsumerCallback consumerCallback) { - return CompletableFuture.supplyAsync(() -> { - try { - DistributedLogManager dlm = namespace.openLog(topic); - log.info("Open stream {} for consumer", topic); - return dlm; - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - }) - .thenApply(dlm -> new DlogBenchmarkConsumer(dlm, consumerCallback)); + String topic, String subscriptionName, ConsumerCallback consumerCallback) { + return CompletableFuture.supplyAsync( + () -> { + try { + DistributedLogManager dlm = namespace.openLog(topic); + log.info("Open stream {} for consumer", topic); + return dlm; + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + }) + .thenApply(dlm -> new DlogBenchmarkConsumer(dlm, consumerCallback)); } @Override @@ -147,5 +143,4 @@ public void close() throws Exception { log.info("BookKeeper benchmark driver successfully shut down"); } - } diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java index ed2f667a3..43f7399a7 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/DlogBenchmarkProducer.java @@ -13,15 +13,14 @@ */ package io.openmessaging.benchmark.driver.bookkeeper; + +import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.Optional; import java.util.concurrent.CompletableFuture; - import org.apache.distributedlog.LogRecord; import org.apache.distributedlog.api.AsyncLogWriter; import org.apache.distributedlog.util.TimeSequencer; -import io.openmessaging.benchmark.driver.BenchmarkProducer; - public class DlogBenchmarkProducer implements BenchmarkProducer { private final AsyncLogWriter writer; @@ -39,10 +38,8 @@ public void close() throws Exception { @Override public CompletableFuture sendAsync(Optional key, byte[] payload) { - LogRecord record = new LogRecord( - sequencer.nextId(), payload); + LogRecord record = new LogRecord(sequencer.nextId(), payload); return writer.write(record).thenApply(dlsn -> null); } - } diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java index 61d359969..95d908774 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/CounterAdaptor.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.bookkeeper.stats; + import org.apache.bookkeeper.stats.Counter; class CounterAdaptor implements dlshade.org.apache.bookkeeper.stats.Counter { diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java index 3900ef854..a675a7484 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/GaugeAdaptor.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.bookkeeper.stats; + import org.apache.bookkeeper.stats.Gauge; class GaugeAdaptor implements Gauge { diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java index 4f0c4fbdd..8f3d16680 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/OpStatsLoggerAdaptor.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.bookkeeper.stats; + import dlshade.org.apache.bookkeeper.stats.OpStatsData; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.stats.OpStatsLogger; diff --git a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java index 37337557d..7f59c2db7 100644 --- a/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java +++ b/driver-bookkeeper/src/main/java/io/openmessaging/benchmark/driver/bookkeeper/stats/StatsLoggerAdaptor.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.bookkeeper.stats; + import dlshade.com.google.common.collect.Maps; import java.util.concurrent.ConcurrentMap; import org.apache.bookkeeper.stats.Gauge; @@ -22,7 +23,8 @@ public class StatsLoggerAdaptor implements dlshade.org.apache.bookkeeper.stats.S private final StatsLogger statsLogger; private final ConcurrentMap gauges; - private final ConcurrentMap statsLoggers; + private final ConcurrentMap + statsLoggers; public StatsLoggerAdaptor(StatsLogger statsLogger) { this.statsLogger = statsLogger; @@ -41,14 +43,16 @@ public dlshade.org.apache.bookkeeper.stats.Counter getCounter(String name) { } @Override - public void registerGauge(String name, dlshade.org.apache.bookkeeper.stats.Gauge gauge) { + public void registerGauge( + String name, dlshade.org.apache.bookkeeper.stats.Gauge gauge) { Gauge gaugeAdaptor = new GaugeAdaptor<>(gauge); statsLogger.registerGauge(name, gaugeAdaptor); gauges.put(gauge, gaugeAdaptor); } @Override - public void unregisterGauge(String name, dlshade.org.apache.bookkeeper.stats.Gauge gauge) { + public void unregisterGauge( + String name, dlshade.org.apache.bookkeeper.stats.Gauge gauge) { Gauge gaugeAdaptor = gauges.remove(gauge); if (null != gaugeAdaptor) { statsLogger.unregisterGauge(name, gaugeAdaptor); @@ -64,7 +68,8 @@ public dlshade.org.apache.bookkeeper.stats.StatsLogger scope(String name) { } @Override - public void removeScope(String name, dlshade.org.apache.bookkeeper.stats.StatsLogger dlShadeStatsLogger) { + public void removeScope( + String name, dlshade.org.apache.bookkeeper.stats.StatsLogger dlShadeStatsLogger) { StatsLogger scopedStatsLogger = statsLoggers.remove(dlShadeStatsLogger); if (null != scopedStatsLogger) { statsLogger.removeScope(name, scopedStatsLogger); diff --git a/driver-jms/README.md b/driver-jms/README.md index 360c6ebba..3792d8bc3 100644 --- a/driver-jms/README.md +++ b/driver-jms/README.md @@ -13,10 +13,12 @@ Rather than simply dropping a JMS Client Library into `/opt/benchmark/lib` the l Follow these instructions to compile the openmessaging benchmark for Fast JMS for Apache Pulsar - Build the openmessaging benchmark package as you would normally + ``` mvn clean package ``` - Run the repacking script + ``` bash driver-jms/package-pulsar.sh ``` @@ -28,6 +30,7 @@ You can now deploy to AWS from `driver-pulsar/deploy`. Follow the [Confluent instructions][1] to create a fat jar. - Create a directory + ``` cd ~ mkdir kafka-jms-client @@ -36,6 +39,7 @@ Follow the [Confluent instructions][1] to create a fat jar. - Create the pom.xml - Change `http://packages.confluent.io/maven/` to `https://packages.confluent.io/maven/` - Build the fat jar + ``` mvn clean package ``` @@ -43,10 +47,12 @@ Follow the [Confluent instructions][1] to create a fat jar. Follow these instructions to compile the openmessaging benchmark for Confluent JMS Client - Build the openmessaging benchmark package + ``` mvn clean package ``` - Run the repacking script passing in the location of the fat jar. EG. `~/kafka-jms-client/target/kafka-jms-client-fat-6.2.1.jar` + ``` bash driver-jms/package-kafka.sh /path/to/the/kafka-jms-client.jar ``` @@ -60,6 +66,5 @@ For Pulsar JMS (and likely Kafka) you will likely want to allocate additional co - Edit your `terraform.tfvars` file to adjust `num_instances["client"]`. - Run `bin/benchmark` with the `--extra` option to allocate more workers as consumers. +[1]: https://docs.confluent.io/platform/current/clients/kafka-jms-client/installation.html#appendix-1 - -[1]: https://docs.confluent.io/platform/current/clients/kafka-jms-client/installation.html#appendix-1 \ No newline at end of file diff --git a/driver-jms/pom.xml b/driver-jms/pom.xml index b8cac2434..12101a80f 100644 --- a/driver-jms/pom.xml +++ b/driver-jms/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - driver-jms - - 2.0.3 - + driver-jms + + 2.0.3 + - - - ${project.groupId} - driver-api - ${project.version} - - - jakarta.jms - jakarta.jms-api - ${jms.version} - - - com.google.guava - guava - - + + + ${project.groupId} + driver-api + ${project.version} + + + com.google.guava + guava + + + jakarta.jms + jakarta.jms-api + ${jms.version} + + diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java index ba2a0f520..e88fe86dc 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkConsumer.java @@ -13,18 +13,17 @@ */ package io.openmessaging.benchmark.driver.jms; + +import io.openmessaging.benchmark.driver.BenchmarkConsumer; +import io.openmessaging.benchmark.driver.ConsumerCallback; import javax.jms.BytesMessage; import javax.jms.Connection; import javax.jms.Message; import javax.jms.MessageConsumer; import javax.jms.Session; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.openmessaging.benchmark.driver.BenchmarkConsumer; -import io.openmessaging.benchmark.driver.ConsumerCallback; - public class JMSBenchmarkConsumer implements BenchmarkConsumer { private final Connection connection; @@ -32,33 +31,39 @@ public class JMSBenchmarkConsumer implements BenchmarkConsumer { private final MessageConsumer consumer; private final boolean useGetBody; - public JMSBenchmarkConsumer(Connection connection, + public JMSBenchmarkConsumer( + Connection connection, Session session, - MessageConsumer consumer, ConsumerCallback callback, - boolean useGetBody) throws Exception { + MessageConsumer consumer, + ConsumerCallback callback, + boolean useGetBody) + throws Exception { this.connection = connection; this.consumer = consumer; this.session = session; this.useGetBody = useGetBody; - consumer.setMessageListener(message -> { - try { - byte[] payload = getPayload(message); - callback.messageReceived(payload, message.getLongProperty("E2EStartMillis")); - message.acknowledge(); - } catch (Throwable e) { - log.warn("Failed to acknowledge message", e); - } - }); + consumer.setMessageListener( + message -> { + try { + byte[] payload = getPayload(message); + callback.messageReceived(payload, message.getLongProperty("E2EStartMillis")); + message.acknowledge(); + } catch (Throwable e) { + log.warn("Failed to acknowledge message", e); + } + }); // Kafka JMS client does not allow you to add a listener after the connection has been started connection.start(); } @Override public void close() throws Exception { - // This exception may be thrown: java.util.concurrent.ExecutionException: java.util.ConcurrentModificationException: KafkaConsumer is not safe for multi-threaded access - // See https://jakarta.ee/specifications/platform/8/apidocs/javax/jms/session#close-- - // and https://jakarta.ee/specifications/platform/8/apidocs/javax/jms/connection#close-- - // It should be enough to just close the connection. + // This exception may be thrown: java.util.concurrent.ExecutionException: + // java.util.ConcurrentModificationException: KafkaConsumer is not safe for multi-threaded + // access + // See https://jakarta.ee/specifications/platform/8/apidocs/javax/jms/session#close-- + // and https://jakarta.ee/specifications/platform/8/apidocs/javax/jms/connection#close-- + // It should be enough to just close the connection. connection.close(); } diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java index ef20d38d5..221c30bc3 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkDriver.java @@ -13,6 +13,16 @@ */ package io.openmessaging.benchmark.driver.jms; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.openmessaging.benchmark.driver.BenchmarkConsumer; +import io.openmessaging.benchmark.driver.BenchmarkDriver; +import io.openmessaging.benchmark.driver.BenchmarkProducer; +import io.openmessaging.benchmark.driver.ConsumerCallback; +import io.openmessaging.benchmark.driver.jms.config.JMSConfig; import java.io.File; import java.io.IOException; import java.io.StringReader; @@ -21,28 +31,16 @@ import java.util.Properties; import java.util.Random; import java.util.concurrent.CompletableFuture; - import javax.jms.Connection; import javax.jms.ConnectionFactory; import javax.jms.Destination; import javax.jms.MessageConsumer; import javax.jms.Session; import javax.jms.Topic; - import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import io.openmessaging.benchmark.driver.BenchmarkConsumer; -import io.openmessaging.benchmark.driver.BenchmarkDriver; -import io.openmessaging.benchmark.driver.BenchmarkProducer; -import io.openmessaging.benchmark.driver.ConsumerCallback; -import io.openmessaging.benchmark.driver.jms.config.JMSConfig; - public class JMSBenchmarkDriver implements BenchmarkDriver { private ConnectionFactory connectionFactory; @@ -55,35 +53,46 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I this.config = readConfig(configurationFile); log.info("JMS driver configuration: {}", writer.writeValueAsString(config)); - if (config.delegateForAdminOperationsClassName != null && !config.delegateForAdminOperationsClassName.isEmpty()) { - log.info("Initializing Driver for Admin operations {}", config.delegateForAdminOperationsClassName); - try - { - delegateForAdminOperations = (BenchmarkDriver) Class.forName(config.delegateForAdminOperationsClassName, - true, JMSBenchmarkDriver.class.getClassLoader()) - .getConstructor().newInstance(); + if (config.delegateForAdminOperationsClassName != null + && !config.delegateForAdminOperationsClassName.isEmpty()) { + log.info( + "Initializing Driver for Admin operations {}", + config.delegateForAdminOperationsClassName); + try { + delegateForAdminOperations = + (BenchmarkDriver) + Class.forName( + config.delegateForAdminOperationsClassName, + true, + JMSBenchmarkDriver.class.getClassLoader()) + .getConstructor() + .newInstance(); delegateForAdminOperations.initialize(configurationFile, statsLogger); - } - catch (Throwable e) - { - log.error("Cannot created delegate driver " + config.delegateForAdminOperationsClassName, e); + } catch (Throwable e) { + log.error( + "Cannot created delegate driver " + config.delegateForAdminOperationsClassName, e); throw new IOException(e); } } - try - { + try { connectionFactory = buildConnectionFactory(); connection = connectionFactory.createConnection(); connection.start(); } catch (Throwable t) { - log.error("Cannot initialize connectionFactoryClassName = "+config.connectionFactoryClassName, t); + log.error( + "Cannot initialize connectionFactoryClassName = " + config.connectionFactoryClassName, t); throw new IOException(t); } } private ConnectionFactory buildConnectionFactory() throws Exception { - Class clazz = (Class) Class.forName(config.connectionFactoryClassName, true, Thread.currentThread().getContextClassLoader()); + Class clazz = + (Class) + Class.forName( + config.connectionFactoryClassName, + true, + Thread.currentThread().getContextClassLoader()); // constructor with a String (like DataStax Pulsar JMS) try { @@ -97,7 +106,8 @@ private ConnectionFactory buildConnectionFactory() throws Exception { Constructor constructor = clazz.getConstructor(Properties.class); Properties props = new Properties(); ObjectMapper mapper = new ObjectMapper(); - Map map = mapper.readValue(new StringReader(config.connectionFactoryConfigurationParam), Map.class); + Map map = + mapper.readValue(new StringReader(config.connectionFactoryConfigurationParam), Map.class); props.putAll(map); return constructor.newInstance(props); } catch (NoSuchMethodException ignore) { @@ -126,11 +136,14 @@ public CompletableFuture createTopic(String topic, int partitions) { public CompletableFuture createProducer(String topic) { try { if (config.sendWithTransactions) { - return CompletableFuture.completedFuture(new JMSBenchmarkTransactionProducer(connection, topic, config.use20api, config.properties)); + return CompletableFuture.completedFuture( + new JMSBenchmarkTransactionProducer( + connection, topic, config.use20api, config.properties)); } else { Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE); Destination destination = session.createTopic(topic); - return CompletableFuture.completedFuture(new JMSBenchmarkProducer(session, destination, config.use20api, config.properties)); + return CompletableFuture.completedFuture( + new JMSBenchmarkProducer(session, destination, config.use20api, config.properties)); } } catch (Exception err) { CompletableFuture res = new CompletableFuture<>(); @@ -140,22 +153,28 @@ public CompletableFuture createProducer(String topic) { } @Override - public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { try { - String selector = config.messageSelector != null && !config.messageSelector.isEmpty() ? config.messageSelector : null; + String selector = + config.messageSelector != null && !config.messageSelector.isEmpty() + ? config.messageSelector + : null; Connection connection = connectionFactory.createConnection(); Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE); Topic destination = session.createTopic(topic); MessageConsumer durableConsumer; if (config.use20api) { - durableConsumer = session.createSharedDurableConsumer(destination, subscriptionName, selector); + durableConsumer = + session.createSharedDurableConsumer(destination, subscriptionName, selector); } else { // in JMS 1.0 we should use session.createDurableSubscriber() // but it is not supported in Confluent Kafka JMS client durableConsumer = session.createConsumer(destination, selector); } - return CompletableFuture.completedFuture(new JMSBenchmarkConsumer(connection, session, durableConsumer, consumerCallback, config.use20api)); + return CompletableFuture.completedFuture( + new JMSBenchmarkConsumer( + connection, session, durableConsumer, consumerCallback, config.use20api)); } catch (Exception err) { CompletableFuture res = new CompletableFuture<>(); res.completeExceptionally(err); @@ -178,7 +197,8 @@ public void close() throws Exception { } } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static JMSConfig readConfig(File configurationFile) throws IOException { diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java index 2a1718066..b6ae05a69 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkProducer.java @@ -13,11 +13,13 @@ */ package io.openmessaging.benchmark.driver.jms; + +import io.openmessaging.benchmark.driver.BenchmarkProducer; +import io.openmessaging.benchmark.driver.jms.config.JMSConfig; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; - import javax.jms.BytesMessage; import javax.jms.CompletionListener; import javax.jms.Destination; @@ -25,13 +27,9 @@ import javax.jms.Message; import javax.jms.MessageProducer; import javax.jms.Session; - -import io.openmessaging.benchmark.driver.jms.config.JMSConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.openmessaging.benchmark.driver.BenchmarkProducer; - public class JMSBenchmarkProducer implements BenchmarkProducer { private final Session session; @@ -39,7 +37,13 @@ public class JMSBenchmarkProducer implements BenchmarkProducer { private final MessageProducer producer; private final boolean useAsyncSend; private final List properties; - public JMSBenchmarkProducer(Session session, Destination destination, boolean useAsyncSend, List properties) throws Exception { + + public JMSBenchmarkProducer( + Session session, + Destination destination, + boolean useAsyncSend, + List properties) + throws Exception { this.session = session; this.destination = destination; this.useAsyncSend = useAsyncSend; @@ -55,36 +59,33 @@ public void close() throws Exception { @Override public CompletableFuture sendAsync(Optional key, byte[] payload) { CompletableFuture res = new CompletableFuture<>(); - try - { + try { BytesMessage bytesMessage = session.createBytesMessage(); bytesMessage.writeBytes(payload); - if (key.isPresent()) - { + if (key.isPresent()) { // a behaviour similar to https://activemq.apache.org/message-groups bytesMessage.setStringProperty("JMSXGroupID", key.get()); } for (JMSConfig.AddProperty prop : properties) { bytesMessage.setStringProperty(prop.name, prop.value); } - // Add a timer property for end to end - bytesMessage.setLongProperty("E2EStartMillis",System.currentTimeMillis()); + // Add a timer property for end to end + bytesMessage.setLongProperty("E2EStartMillis", System.currentTimeMillis()); if (useAsyncSend) { - producer.send(bytesMessage, new CompletionListener() - { - @Override - public void onCompletion(Message message) - { - res.complete(null); - } + producer.send( + bytesMessage, + new CompletionListener() { + @Override + public void onCompletion(Message message) { + res.complete(null); + } - @Override - public void onException(Message message, Exception exception) - { - log.error("send completed with error", exception); - res.completeExceptionally(exception); - } - }); + @Override + public void onException(Message message, Exception exception) { + log.error("send completed with error", exception); + res.completeExceptionally(exception); + } + }); } else { producer.send(bytesMessage); res.complete(null); @@ -94,5 +95,6 @@ public void onException(Message message, Exception exception) } return res; } + private static final Logger log = LoggerFactory.getLogger(JMSBenchmarkProducer.class); } diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java index fc3ca70a6..01087a9cb 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/JMSBenchmarkTransactionProducer.java @@ -13,17 +13,23 @@ */ package io.openmessaging.benchmark.driver.jms; + import io.openmessaging.benchmark.driver.BenchmarkProducer; import io.openmessaging.benchmark.driver.jms.config.JMSConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.jms.*; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import javax.jms.BytesMessage; +import javax.jms.CompletionListener; +import javax.jms.Connection; +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.MessageProducer; +import javax.jms.Session; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class JMSBenchmarkTransactionProducer implements BenchmarkProducer { @@ -31,7 +37,13 @@ public class JMSBenchmarkTransactionProducer implements BenchmarkProducer { private final boolean useAsyncSend; private final Connection connection; private final List properties; - public JMSBenchmarkTransactionProducer(Connection connection, String destination, boolean useAsyncSend, List properties) throws Exception { + + public JMSBenchmarkTransactionProducer( + Connection connection, + String destination, + boolean useAsyncSend, + List properties) + throws Exception { this.destination = destination; this.useAsyncSend = useAsyncSend; this.connection = connection; @@ -39,57 +51,54 @@ public JMSBenchmarkTransactionProducer(Connection connection, String destination } @Override - public void close() { - } + public void close() {} @Override public CompletableFuture sendAsync(Optional key, byte[] payload) { - try - { + try { // start a new Session every time, we cannot share the same Session // among the Producers because we want to have control over the commit operation Session session = connection.createSession(true, Session.SESSION_TRANSACTED); MessageProducer producer = session.createProducer(session.createTopic(destination)); BytesMessage bytesMessage = session.createBytesMessage(); bytesMessage.writeBytes(payload); - if (key.isPresent()) - { + if (key.isPresent()) { // a behaviour similar to https://activemq.apache.org/message-groups bytesMessage.setStringProperty("JMSXGroupID", key.get()); } for (JMSConfig.AddProperty prop : properties) { bytesMessage.setStringProperty(prop.name, prop.value); } - // Add a timer property for end to end - bytesMessage.setLongProperty("E2EStartMillis",System.currentTimeMillis()); + // Add a timer property for end to end + bytesMessage.setLongProperty("E2EStartMillis", System.currentTimeMillis()); if (useAsyncSend) { CompletableFuture res = new CompletableFuture<>(); - producer.send(bytesMessage, new CompletionListener() - { - @Override - public void onCompletion(Message message) - { - res.complete(null); - } + producer.send( + bytesMessage, + new CompletionListener() { + @Override + public void onCompletion(Message message) { + res.complete(null); + } - @Override - public void onException(Message message, Exception exception) - { - log.info("send completed with error", exception); - res.completeExceptionally(exception); - } - }); - return res.whenCompleteAsync((msg, error) -> { - if (error == null) { - // you cannot close the producer and session inside the CompletionListener - try { - session.commit(); - } catch (JMSException err) { - throw new CompletionException(err); - } - } - ensureClosed(producer, session);; - }); + @Override + public void onException(Message message, Exception exception) { + log.info("send completed with error", exception); + res.completeExceptionally(exception); + } + }); + return res.whenCompleteAsync( + (msg, error) -> { + if (error == null) { + // you cannot close the producer and session inside the CompletionListener + try { + session.commit(); + } catch (JMSException err) { + throw new CompletionException(err); + } + } + ensureClosed(producer, session); + }); } else { try { @@ -107,10 +116,9 @@ public void onException(Message message, Exception exception) res.completeExceptionally(err); return res; } - } - private void ensureClosed(MessageProducer producer, Session session) { + private void ensureClosed(MessageProducer producer, Session session) { try { producer.close(); } catch (Throwable err) { diff --git a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java index e550872d5..3b936024c 100644 --- a/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java +++ b/driver-jms/src/main/java/io/openmessaging/benchmark/driver/jms/config/JMSConfig.java @@ -14,11 +14,11 @@ package io.openmessaging.benchmark.driver.jms.config; + import java.util.ArrayList; import java.util.List; -public class JMSConfig -{ +public class JMSConfig { public String connectionFactoryClassName = ""; public String connectionFactoryConfigurationParam = ""; diff --git a/driver-kafka/deploy/README.md b/driver-kafka/deploy/README.md index 67f1fadbf..9cadfd6c9 100644 --- a/driver-kafka/deploy/README.md +++ b/driver-kafka/deploy/README.md @@ -9,8 +9,8 @@ Customize the instance types in the terraform.tfvars. If you choose larger instances, they come with more drives. To include those in the benchmarks you must: - - update the Ansible script to include them in the mount and filesystem tasks - - update the server.properties to include them in the logs.dir config +- update the Ansible script to include them in the mount and filesystem tasks +- update the server.properties to include them in the logs.dir config NOTE: When using d2 instances, the instance stores are not automatically generated. You must add them to the provision-kafka-aws.tf file. @@ -34,4 +34,4 @@ For instructions on how to run a benchmark see the [Kafka instructions](http://o When using 4 client VMs or less you may see lower throughput when using compression. Compression is performed by the producers and consumers only (when using defaults) and clients need to be spread across more VMs to see any throughput gains. -Obviously, throughput may not be your primary goal when using compression. \ No newline at end of file +Obviously, throughput may not be your primary goal when using compression. diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index 6619a4cc9..176bdfff4 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - - driver-kafka + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + + driver-kafka - - - ${project.groupId} - driver-api - ${project.version} - - - org.apache.kafka - kafka-clients - 3.2.1 - - - org.projectlombok - lombok - - - org.assertj - assertj-core - - - org.apache.logging.log4j - log4j-slf4j-impl - test - - - org.junit.jupiter - junit-jupiter - - - org.mockito - mockito-junit-jupiter - - + + + ${project.groupId} + driver-api + ${project.version} + + + org.apache.kafka + kafka-clients + 3.2.1 + + + org.projectlombok + lombok + provided + + + org.apache.logging.log4j + log4j-slf4j-impl + test + + + org.assertj + assertj-core + test + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-junit-jupiter + test + + diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java index 171c852e5..e07b417d9 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkConsumer.java @@ -13,6 +13,9 @@ */ package io.openmessaging.benchmark.driver.kafka; + +import io.openmessaging.benchmark.driver.BenchmarkConsumer; +import io.openmessaging.benchmark.driver.ConsumerCallback; import java.time.Duration; import java.util.HashMap; import java.util.Map; @@ -20,16 +23,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; - import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; - -import io.openmessaging.benchmark.driver.BenchmarkConsumer; -import io.openmessaging.benchmark.driver.ConsumerCallback; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,41 +43,50 @@ public class KafkaBenchmarkConsumer implements BenchmarkConsumer { private volatile boolean closing = false; private boolean autoCommit; - public KafkaBenchmarkConsumer(KafkaConsumer consumer, - Properties consumerConfig, - ConsumerCallback callback) { + public KafkaBenchmarkConsumer( + KafkaConsumer consumer, + Properties consumerConfig, + ConsumerCallback callback) { this(consumer, consumerConfig, callback, 100L); } - public KafkaBenchmarkConsumer(KafkaConsumer consumer, - Properties consumerConfig, - ConsumerCallback callback, - long pollTimeoutMs) { + public KafkaBenchmarkConsumer( + KafkaConsumer consumer, + Properties consumerConfig, + ConsumerCallback callback, + long pollTimeoutMs) { this.consumer = consumer; this.executor = Executors.newSingleThreadExecutor(); - this.autoCommit= Boolean.valueOf((String)consumerConfig.getOrDefault(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false")); - this.consumerTask = this.executor.submit(() -> { - while (!closing) { - try { - ConsumerRecords records = consumer.poll(Duration.ofMillis(pollTimeoutMs)); + this.autoCommit = + Boolean.valueOf( + (String) + consumerConfig.getOrDefault(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false")); + this.consumerTask = + this.executor.submit( + () -> { + while (!closing) { + try { + ConsumerRecords records = + consumer.poll(Duration.ofMillis(pollTimeoutMs)); - Map offsetMap = new HashMap<>(); - for (ConsumerRecord record : records) { - callback.messageReceived(record.value(), record.timestamp()); + Map offsetMap = new HashMap<>(); + for (ConsumerRecord record : records) { + callback.messageReceived(record.value(), record.timestamp()); - offsetMap.put(new TopicPartition(record.topic(), record.partition()), - new OffsetAndMetadata(record.offset()+1)); - } + offsetMap.put( + new TopicPartition(record.topic(), record.partition()), + new OffsetAndMetadata(record.offset() + 1)); + } - if (!autoCommit&&!offsetMap.isEmpty()) { - // Async commit all messages polled so far - consumer.commitAsync(offsetMap, null); - } - } catch(Exception e){ - log.error("exception occur while consuming message", e); - } - } - }); + if (!autoCommit && !offsetMap.isEmpty()) { + // Async commit all messages polled so far + consumer.commitAsync(offsetMap, null); + } + } catch (Exception e) { + log.error("exception occur while consuming message", e); + } + } + }); } @Override @@ -88,5 +96,4 @@ public void close() throws Exception { consumerTask.get(); consumer.close(); } - } diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java index 605d42257..a2789ac2a 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkDriver.java @@ -13,6 +13,14 @@ */ package io.openmessaging.benchmark.driver.kafka; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.openmessaging.benchmark.driver.BenchmarkConsumer; +import io.openmessaging.benchmark.driver.BenchmarkDriver; +import io.openmessaging.benchmark.driver.BenchmarkProducer; +import io.openmessaging.benchmark.driver.ConsumerCallback; import java.io.File; import java.io.IOException; import java.io.StringReader; @@ -24,7 +32,6 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.CompletableFuture; - import org.apache.bookkeeper.stats.StatsLogger; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -36,15 +43,6 @@ import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - -import io.openmessaging.benchmark.driver.BenchmarkConsumer; -import io.openmessaging.benchmark.driver.BenchmarkDriver; -import io.openmessaging.benchmark.driver.BenchmarkProducer; -import io.openmessaging.benchmark.driver.ConsumerCallback; - public class KafkaBenchmarkDriver implements BenchmarkDriver { private Config config; @@ -68,14 +66,18 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I producerProperties = new Properties(); commonProperties.forEach((key, value) -> producerProperties.put(key, value)); producerProperties.load(new StringReader(config.producerConfig)); - producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + producerProperties.put( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + producerProperties.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); consumerProperties = new Properties(); commonProperties.forEach((key, value) -> consumerProperties.put(key, value)); consumerProperties.load(new StringReader(config.consumerConfig)); - consumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - consumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerProperties.put( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + consumerProperties.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); topicProperties = new Properties(); topicProperties.load(new StringReader(config.topicConfig)); @@ -97,7 +99,8 @@ public CompletableFuture createTopic(String topic, int partitions) { public CompletableFuture createTopics(List topicInfos) { @SuppressWarnings({"unchecked", "rawtypes"}) Map topicConfigs = new HashMap<>((Map) topicProperties); - KafkaTopicCreator topicCreator = new KafkaTopicCreator(admin, topicConfigs, config.replicationFactor); + KafkaTopicCreator topicCreator = + new KafkaTopicCreator(admin, topicConfigs, config.replicationFactor); return topicCreator.create(topicInfos); } @@ -118,22 +121,22 @@ public CompletableFuture createProducer(String topic) { } @Override - public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { Properties properties = new Properties(); consumerProperties.forEach((key, value) -> properties.put(key, value)); properties.put(ConsumerConfig.GROUP_ID_CONFIG, subscriptionName); KafkaConsumer consumer = new KafkaConsumer<>(properties); try { consumer.subscribe(Arrays.asList(topic)); - return CompletableFuture.completedFuture(new KafkaBenchmarkConsumer(consumer,consumerProperties,consumerCallback)); + return CompletableFuture.completedFuture( + new KafkaBenchmarkConsumer(consumer, consumerProperties, consumerCallback)); } catch (Throwable t) { consumer.close(); CompletableFuture future = new CompletableFuture<>(); future.completeExceptionally(t); return future; } - } @Override @@ -148,6 +151,7 @@ public void close() throws Exception { admin.close(); } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java index 30c10b509..6c62d9fbc 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaBenchmarkProducer.java @@ -26,15 +26,13 @@ */ package io.openmessaging.benchmark.driver.kafka; + +import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.Optional; import java.util.concurrent.CompletableFuture; - -import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; -import io.openmessaging.benchmark.driver.BenchmarkProducer; - public class KafkaBenchmarkProducer implements BenchmarkProducer { private final Producer producer; @@ -51,13 +49,15 @@ public CompletableFuture sendAsync(Optional key, byte[] payload) { CompletableFuture future = new CompletableFuture<>(); - producer.send(record, (metadata, exception) -> { - if (exception != null) { - future.completeExceptionally(exception); - } else { - future.complete(null); - } - }); + producer.send( + record, + (metadata, exception) -> { + if (exception != null) { + future.completeExceptionally(exception); + } else { + future.complete(null); + } + }); return future; } @@ -66,5 +66,4 @@ public CompletableFuture sendAsync(Optional key, byte[] payload) { public void close() throws Exception { producer.close(); } - } diff --git a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java index d114d1521..82d3e38b7 100644 --- a/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java +++ b/driver-kafka/src/main/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreator.java @@ -17,6 +17,7 @@ import static java.util.function.Function.identity; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; + import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo; import java.util.ArrayList; import java.util.List; @@ -60,21 +61,26 @@ private void createBlocking(List topicInfos) { AtomicInteger succeeded = new AtomicInteger(); ScheduledFuture loggingFuture = - executor.scheduleAtFixedRate(() -> log.info("Created topics {}/{}", succeeded.get(), topicInfos.size()), - 10, 10, SECONDS); + executor.scheduleAtFixedRate( + () -> log.info("Created topics {}/{}", succeeded.get(), topicInfos.size()), + 10, + 10, + SECONDS); try { while (succeeded.get() < topicInfos.size()) { int batchSize = queue.drainTo(batch, maxBatchSize); if (batchSize > 0) { - executeBatch(batch).forEach((topicInfo, success) -> { - if (success) { - succeeded.incrementAndGet(); - } else { - //noinspection ResultOfMethodCallIgnored - queue.offer(topicInfo); - } - }); + executeBatch(batch) + .forEach( + (topicInfo, success) -> { + if (success) { + succeeded.incrementAndGet(); + } else { + //noinspection ResultOfMethodCallIgnored + queue.offer(topicInfo); + } + }); batch.clear(); } } @@ -85,20 +91,17 @@ private void createBlocking(List topicInfos) { private Map executeBatch(List batch) { log.debug("Executing batch, size: {}", batch.size()); - Map lookup = batch.stream() - .collect(toMap(TopicInfo::getTopic, identity())); + Map lookup = batch.stream().collect(toMap(TopicInfo::getTopic, identity())); - List newTopics = batch.stream() - .map(this::newTopic) - .collect(toList()); + List newTopics = batch.stream().map(this::newTopic).collect(toList()); - return admin.createTopics(newTopics).values() - .entrySet().stream() + return admin.createTopics(newTopics).values().entrySet().stream() .collect(toMap(e -> lookup.get(e.getKey()), e -> isSuccess(e.getValue()))); } private NewTopic newTopic(TopicInfo topicInfo) { - NewTopic newTopic = new NewTopic(topicInfo.getTopic(), topicInfo.getPartitions(), replicationFactor); + NewTopic newTopic = + new NewTopic(topicInfo.getTopic(), topicInfo.getPartitions(), replicationFactor); newTopic.configs(topicConfigs); return newTopic; } diff --git a/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java b/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java index ca5b084dc..25b38a0e1 100644 --- a/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java +++ b/driver-kafka/src/test/java/io/openmessaging/benchmark/driver/kafka/KafkaTopicCreatorTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; + import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo; import java.util.HashMap; import java.util.List; @@ -46,12 +47,9 @@ class KafkaTopicCreatorTest { private final int partitions = 1; private final short replicationFactor = 1; private final TopicInfo topicInfo = new TopicInfo(topic, partitions); - @Mock - private AdminClient admin; - @Mock - private CreateTopicsResult createTopicsResult; - @Captor - private ArgumentCaptor> captor; + @Mock private AdminClient admin; + @Mock private CreateTopicsResult createTopicsResult; + @Captor private ArgumentCaptor> captor; private KafkaTopicCreator topicCreator; @BeforeEach @@ -121,4 +119,4 @@ private void assertNewTopics(List newTopics) { assertThat(newTopic.replicationFactor()).isEqualTo(replicationFactor); assertThat(newTopic.configs()).isSameAs(topicConfigs); } -} \ No newline at end of file +} diff --git a/driver-kop/pom.xml b/driver-kop/pom.xml index 5770fdfe2..1904ed9d4 100644 --- a/driver-kop/pom.xml +++ b/driver-kop/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - driver-kop + driver-kop - - - ${project.groupId} - driver-api - ${project.version} - - - ${project.groupId} - driver-kafka - ${project.version} - - - ${project.groupId} - driver-pulsar - ${project.version} - - - io.streamnative.pulsar.handlers - kafka-payload-processor - 2.10.1.7 - - - org.apache.commons - commons-lang3 - - - org.testng - testng - 7.6.1 - test - - + + + ${project.groupId} + driver-api + ${project.version} + + + ${project.groupId} + driver-kafka + ${project.version} + + + ${project.groupId} + driver-pulsar + ${project.version} + + + io.streamnative.pulsar.handlers + kafka-payload-processor + 2.10.1.7 + + + org.apache.commons + commons-lang3 + + + org.testng + testng + 7.6.1 + test + + diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java index d855d7c84..32ccb6ba2 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriver.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.kop; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -61,8 +62,9 @@ public class KopBenchmarkDriver implements BenchmarkDriver { - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private final List producers = new CopyOnWriteArrayList<>(); private final List consumers = new CopyOnWriteArrayList<>(); @@ -93,23 +95,30 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I consumerProperties = new Properties(); commonProperties.forEach(consumerProperties::put); consumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - consumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); + consumerProperties.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); final PulsarConfig pulsarConfig = config.pulsarConfig; if (config.producerType.equals(ClientType.PULSAR)) { - producerBuilder = getPulsarClient(pulsarConfig.serviceUrl).newProducer() - .enableBatching(pulsarConfig.batchingEnabled) - .blockIfQueueFull(pulsarConfig.blockIfQueueFull) - .batchingMaxPublishDelay(pulsarConfig.batchingMaxPublishDelayMs, TimeUnit.MILLISECONDS) - .batchingMaxBytes(pulsarConfig.batchingMaxBytes) - .maxPendingMessages(pulsarConfig.pendingQueueSize) - .maxPendingMessagesAcrossPartitions(pulsarConfig.maxPendingMessagesAcrossPartitions); + producerBuilder = + getPulsarClient(pulsarConfig.serviceUrl) + .newProducer() + .enableBatching(pulsarConfig.batchingEnabled) + .blockIfQueueFull(pulsarConfig.blockIfQueueFull) + .batchingMaxPublishDelay( + pulsarConfig.batchingMaxPublishDelayMs, TimeUnit.MILLISECONDS) + .batchingMaxBytes(pulsarConfig.batchingMaxBytes) + .maxPendingMessages(pulsarConfig.pendingQueueSize) + .maxPendingMessagesAcrossPartitions(pulsarConfig.maxPendingMessagesAcrossPartitions); } if (config.consumerType.equals(ClientType.PULSAR)) { - consumerBuilder = getPulsarClient(pulsarConfig.serviceUrl).newConsumer(Schema.BYTEBUFFER) - .subscriptionType(SubscriptionType.Failover) - .receiverQueueSize(pulsarConfig.receiverQueueSize) - .maxTotalReceiverQueueSizeAcrossPartitions(pulsarConfig.maxTotalReceiverQueueSizeAcrossPartitions); + consumerBuilder = + getPulsarClient(pulsarConfig.serviceUrl) + .newConsumer(Schema.BYTEBUFFER) + .subscriptionType(SubscriptionType.Failover) + .receiverQueueSize(pulsarConfig.receiverQueueSize) + .maxTotalReceiverQueueSizeAcrossPartitions( + pulsarConfig.maxTotalReceiverQueueSizeAcrossPartitions); } } @@ -123,13 +132,17 @@ public CompletableFuture createTopic(String topic, int partitions) { // replicationFactor is meaningless in KoP final NewTopic newTopic = new NewTopic(topic, partitions, (short) 1L); final CompletableFuture future = new CompletableFuture<>(); - admin.createTopics(Collections.singletonList(newTopic)).all().whenComplete((result, throwable) -> { - if (throwable == null) { - future.complete(result); - } else { - future.completeExceptionally(throwable); - } - }); + admin + .createTopics(Collections.singletonList(newTopic)) + .all() + .whenComplete( + (result, throwable) -> { + if (throwable == null) { + future.complete(result); + } else { + future.completeExceptionally(throwable); + } + }); return future; } @@ -141,12 +154,17 @@ public CompletableFuture createProducer(String topic) { producers.add(producer); return CompletableFuture.completedFuture(producer); } else if (config.consumerType.equals(ClientType.PULSAR)) { - return producerBuilder.clone().topic(topic).createAsync().thenApply(PulsarBenchmarkProducer::new); + return producerBuilder + .clone() + .topic(topic) + .createAsync() + .thenApply(PulsarBenchmarkProducer::new); } else { throw new IllegalArgumentException("producerType " + config.producerType + " is invalid"); } } + @SuppressWarnings("checkstyle:LineLength") @Override public CompletableFuture createConsumer( String topic, String subscriptionName, ConsumerCallback consumerCallback) { @@ -157,19 +175,27 @@ public CompletableFuture createConsumer( properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); final KafkaConsumer kafkaConsumer = new KafkaConsumer<>(properties); kafkaConsumer.subscribe(Collections.singleton(topic)); - final BenchmarkConsumer consumer = new KafkaBenchmarkConsumer( - kafkaConsumer, properties, consumerCallback, config.pollTimeoutMs); + final BenchmarkConsumer consumer = + new KafkaBenchmarkConsumer( + kafkaConsumer, properties, consumerCallback, config.pollTimeoutMs); consumers.add(consumer); return CompletableFuture.completedFuture(consumer); } else if (config.consumerType.equals(ClientType.PULSAR)) { final List>> futures = new ArrayList<>(); - return client.getPartitionsForTopic(topic) - .thenCompose(partitions -> { - partitions.forEach(p -> futures.add( - createInternalPulsarConsumer(p, subscriptionName, consumerCallback))); - return FutureUtil.waitForAll(futures); - }).thenApply(__ -> new PulsarBenchmarkConsumer(futures.stream().map(CompletableFuture::join) - .collect(Collectors.toList()))); + return client + .getPartitionsForTopic(topic) + .thenCompose( + partitions -> { + partitions.forEach( + p -> + futures.add( + createInternalPulsarConsumer(p, subscriptionName, consumerCallback))); + return FutureUtil.waitForAll(futures); + }) + .thenApply( + __ -> + new PulsarBenchmarkConsumer( + futures.stream().map(CompletableFuture::join).collect(Collectors.toList()))); } else { throw new IllegalArgumentException("consumerType " + config.consumerType + " is invalid"); } @@ -198,19 +224,22 @@ private PulsarClient getPulsarClient(String serviceUrl) throws PulsarClientExcep private CompletableFuture> createInternalPulsarConsumer( String topic, String subscriptionName, ConsumerCallback callback) { - return consumerBuilder.clone() + return consumerBuilder + .clone() .topic(topic) .subscriptionName(subscriptionName) - .messagePayloadProcessor(new KafkaPayloadProcessor()) // support consuming Kafka format messages + .messagePayloadProcessor( + new KafkaPayloadProcessor()) // support consuming Kafka format messages .poolMessages(true) - .messageListener((c, msg) -> { - try { - callback.messageReceived(msg.getValue(), msg.getPublishTime()); - c.acknowledgeAsync(msg); - } finally { - msg.release(); - } - }) + .messageListener( + (c, msg) -> { + try { + callback.messageReceived(msg.getValue(), msg.getPublishTime()); + c.acknowledgeAsync(msg); + } finally { + msg.release(); + } + }) .subscribeAsync(); } } diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java index 1e25a856e..0f279f645 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/ClientType.java @@ -13,9 +13,12 @@ */ package io.openmessaging.benchmark.driver.kop.config; + import com.fasterxml.jackson.annotation.JsonProperty; public enum ClientType { - @JsonProperty("kafka") KAFKA, - @JsonProperty("pulsar") PULSAR + @JsonProperty("kafka") + KAFKA, + @JsonProperty("pulsar") + PULSAR } diff --git a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java index daff56a9c..36cc4b0fe 100644 --- a/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java +++ b/driver-kop/src/main/java/io/openmessaging/benchmark/driver/kop/config/Config.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.kop.config; + import java.io.IOException; import java.io.StringReader; import java.util.Properties; diff --git a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java index 59e6d9156..277bdb146 100644 --- a/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java +++ b/driver-kop/src/test/java/io/openmessaging/benchmark/driver/kop/KopBenchmarkDriverTest.java @@ -19,14 +19,14 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import io.openmessaging.benchmark.driver.kop.config.ClientType; +import io.openmessaging.benchmark.driver.kop.config.Config; +import io.openmessaging.benchmark.driver.kop.config.PulsarConfig; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; import java.util.Properties; -import io.openmessaging.benchmark.driver.kop.config.ClientType; -import io.openmessaging.benchmark.driver.kop.config.Config; -import io.openmessaging.benchmark.driver.kop.config.PulsarConfig; import org.testng.annotations.Test; public class KopBenchmarkDriverTest { diff --git a/driver-nats-streaming/pom.xml b/driver-nats-streaming/pom.xml index 40a876e2e..a589c5277 100644 --- a/driver-nats-streaming/pom.xml +++ b/driver-nats-streaming/pom.xml @@ -1,3 +1,4 @@ + - - + 4.0.0 - messaging-benchmark io.openmessaging.benchmark + messaging-benchmark 0.0.1-SNAPSHOT - 4.0.0 driver-nats-streaming - - io.nats - java-nats-streaming - 2.1.0 - ${project.groupId} driver-api ${project.version} + + io.nats + java-nats-streaming + 2.1.0 + - \ No newline at end of file + diff --git a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkConsumer.java b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkConsumer.java index 6483ffc14..1e6e1c1bd 100644 --- a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkConsumer.java +++ b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkConsumer.java @@ -13,19 +13,21 @@ */ package io.openmessaging.benchmark.driver.natsStreaming; + import io.nats.streaming.StreamingConnection; -import io.nats.streaming.Subscription; import io.openmessaging.benchmark.driver.BenchmarkConsumer; public class NatsStreamingBenchmarkConsumer implements BenchmarkConsumer { private StreamingConnection streamingConnection; private boolean unsubscribe; + public NatsStreamingBenchmarkConsumer(StreamingConnection streamingConnection) { this.unsubscribe = false; this.streamingConnection = streamingConnection; } - @Override public void close() throws Exception { + @Override + public void close() throws Exception { if (!unsubscribe) { unsubscribe = true; streamingConnection.close(); diff --git a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkDriver.java b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkDriver.java index 3a213a70e..06e31b988 100644 --- a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkDriver.java +++ b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkDriver.java @@ -14,6 +14,7 @@ package io.openmessaging.benchmark.driver.natsStreaming; import static java.nio.charset.StandardCharsets.UTF_8; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -37,7 +38,6 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.LoggerFactory; - public class NatsStreamingBenchmarkDriver implements BenchmarkDriver { private final String defaultClusterId = "test-cluster"; private String clusterId; @@ -45,7 +45,9 @@ public class NatsStreamingBenchmarkDriver implements BenchmarkDriver { private StreamingConnection natsStreamingPublisher; private SubscriptionOptions.Builder subBuilder = new SubscriptionOptions.Builder(); private Options.Builder optsBuilder = new Options.Builder(); - @Override public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException { + + @Override + public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException { config = mapper.readValue(configurationFile, NatsStreamingClientConfig.class); log.info("read config file," + config.toString()); if (config.clusterId != null) { @@ -62,11 +64,13 @@ public class NatsStreamingBenchmarkDriver implements BenchmarkDriver { subBuilder.maxInFlight(config.maxInFlight); } - @Override public String getTopicNamePrefix() { + @Override + public String getTopicNamePrefix() { return "Nats-streaming-benchmark"; } - @Override public CompletableFuture createTopic(String topic, int partitions) { + @Override + public CompletableFuture createTopic(String topic, int partitions) { log.info("nats streaming create a topic" + topic); log.info("ignore partitions"); CompletableFuture future = new CompletableFuture<>(); @@ -74,7 +78,8 @@ public class NatsStreamingBenchmarkDriver implements BenchmarkDriver { return future; } - @Override public CompletableFuture createProducer(String topic) { + @Override + public CompletableFuture createProducer(String topic) { if (natsStreamingPublisher == null) { String clientId = "ProducerInstance" + getRandomString(); try { @@ -85,43 +90,54 @@ public class NatsStreamingBenchmarkDriver implements BenchmarkDriver { } } - return CompletableFuture.completedFuture(new NatsStreamingBenchmarkProducer(natsStreamingPublisher, topic)); + return CompletableFuture.completedFuture( + new NatsStreamingBenchmarkProducer(natsStreamingPublisher, topic)); } - @Override public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { + @Override + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { Subscription sub; StreamingConnection streamingConnection; String clientId = "ConsumerInstance" + getRandomString(); try { streamingConnection = NatsStreaming.connect(clusterId, clientId, optsBuilder.build()); - streamingConnection.subscribe(topic, subscriptionName, new MessageHandler() { - @Override public void onMessage(Message message) { - consumerCallback.messageReceived(message.getData(), message.getTimestamp()); - } - }, subBuilder.build()); + streamingConnection.subscribe( + topic, + subscriptionName, + new MessageHandler() { + @Override + public void onMessage(Message message) { + consumerCallback.messageReceived(message.getData(), message.getTimestamp()); + } + }, + subBuilder.build()); } catch (Exception e) { log.warn("nats streaming create consumer exception", e); return null; } - return CompletableFuture.completedFuture(new NatsStreamingBenchmarkConsumer(streamingConnection)); + return CompletableFuture.completedFuture( + new NatsStreamingBenchmarkConsumer(streamingConnection)); } - @Override public void close() throws Exception { + @Override + public void close() throws Exception { if (natsStreamingPublisher != null) { natsStreamingPublisher.close(); natsStreamingPublisher = null; } } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - private static final org.slf4j.Logger log = LoggerFactory.getLogger(NatsStreamingBenchmarkDriver.class); + private static final org.slf4j.Logger log = + LoggerFactory.getLogger(NatsStreamingBenchmarkDriver.class); private static final Random random = new Random(); - private static final String getRandomString() { + private static String getRandomString() { byte[] buffer = new byte[5]; random.nextBytes(buffer); return BaseEncoding.base64Url().omitPadding().encode(buffer); @@ -131,24 +147,34 @@ public static void main(String[] args) throws Exception { try { Options opts = new Options.Builder().natsUrl("nats://0.0.0.0:4222").build(); SubscriptionOptions.Builder builder = new SubscriptionOptions.Builder(); - StreamingConnection streamingConnection = NatsStreaming.connect("mycluster", "benchmark-sub", opts); - Subscription sub = streamingConnection.subscribe("topicTest", "subscription", new MessageHandler() { - @Override public void onMessage(Message message) { - System.out.println(message.toString()); - } - }, builder.build()); - StreamingConnection natsStreamingPublisher = NatsStreaming.connect("mycluster", "benchmark-pub", opts); + StreamingConnection streamingConnection = + NatsStreaming.connect("mycluster", "benchmark-sub", opts); + Subscription sub = + streamingConnection.subscribe( + "topicTest", + "subscription", + new MessageHandler() { + @Override + public void onMessage(Message message) { + System.out.println(message.toString()); + } + }, + builder.build()); + StreamingConnection natsStreamingPublisher = + NatsStreaming.connect("mycluster", "benchmark-pub", opts); final String[] guid = new String[1]; - AckHandler acb = new AckHandler() { - @Override public void onAck(String s, Exception e) { - if ((e != null) || !guid[0].equals(s)) { - System.out.println("pub error"); - } else { - System.out.println("pub success"); - } - } - }; + AckHandler acb = + new AckHandler() { + @Override + public void onAck(String s, Exception e) { + if ((e != null) || !guid[0].equals(s)) { + System.out.println("pub error"); + } else { + System.out.println("pub success"); + } + } + }; guid[0] = natsStreamingPublisher.publish("topicTest", "HelloStreaming".getBytes(UTF_8), acb); diff --git a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkProducer.java b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkProducer.java index 06228f6c0..2393c0719 100644 --- a/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkProducer.java +++ b/driver-nats-streaming/src/main/java/io/openmessaging/benchmark/driver/natsStreaming/NatsStreamingBenchmarkProducer.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.natsStreaming; + import io.nats.streaming.AckHandler; import io.nats.streaming.StreamingConnection; import io.openmessaging.benchmark.driver.BenchmarkProducer; @@ -22,23 +23,28 @@ public class NatsStreamingBenchmarkProducer implements BenchmarkProducer { private StreamingConnection natsStreamingPublisher; private String topic; - public NatsStreamingBenchmarkProducer (StreamingConnection natsStreamingPublisher, String topic) { + + public NatsStreamingBenchmarkProducer(StreamingConnection natsStreamingPublisher, String topic) { this.natsStreamingPublisher = natsStreamingPublisher; this.topic = topic; } - @Override public CompletableFuture sendAsync(Optional key, byte[] payload) { + @Override + public CompletableFuture sendAsync(Optional key, byte[] payload) { CompletableFuture future = new CompletableFuture<>(); final String[] guid = new String[1]; - AckHandler acb = new AckHandler() { - @Override public void onAck(String s, Exception e) { - if ((e != null) || !guid[0].equals(s)) { - future.completeExceptionally(e != null ? e : new IllegalStateException("guid != nuid")); - } else { - future.complete(null); - } - } - }; + AckHandler acb = + new AckHandler() { + @Override + public void onAck(String s, Exception e) { + if ((e != null) || !guid[0].equals(s)) { + future.completeExceptionally( + e != null ? e : new IllegalStateException("guid != nuid")); + } else { + future.complete(null); + } + } + }; try { guid[0] = natsStreamingPublisher.publish(topic, payload, acb); } catch (Exception e) { @@ -50,7 +56,6 @@ public NatsStreamingBenchmarkProducer (StreamingConnection natsStreamingPublishe return future; } - @Override public void close() throws Exception { - - } + @Override + public void close() throws Exception {} } diff --git a/driver-nats/pom.xml b/driver-nats/pom.xml index 6518eae9c..d2d6bc6d1 100644 --- a/driver-nats/pom.xml +++ b/driver-nats/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - messaging-benchmark io.openmessaging.benchmark + messaging-benchmark 0.0.1-SNAPSHOT - 4.0.0 driver-nats - - io.nats - jnats - 2.15.6 - ${project.groupId} driver-api ${project.version} + + io.nats + jnats + 2.15.6 + - \ No newline at end of file + diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java index fd44b4a2b..8627475df 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkConsumer.java @@ -13,10 +13,10 @@ */ package io.openmessaging.benchmark.driver.nats; -import io.nats.client.Connection; + import io.openmessaging.benchmark.driver.BenchmarkConsumer; public class NatsBenchmarkConsumer implements BenchmarkConsumer { - @Override public void close() { - } + @Override + public void close() {} } diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index 3de6ea775..aa2989254 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -13,17 +13,17 @@ */ package io.openmessaging.benchmark.driver.nats; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.nats.client.Connection; import io.nats.client.Dispatcher; import io.nats.client.ErrorListener; import io.nats.client.JetStream; import io.nats.client.JetStreamManagement; -import io.nats.client.JetStreamOptions; import io.nats.client.JetStreamSubscription; import io.nats.client.Message; -import io.nats.client.MessageHandler; import io.nats.client.Nats; import io.nats.client.Options; import io.nats.client.PushSubscribeOptions; @@ -31,7 +31,6 @@ import io.nats.client.api.StreamConfiguration; import io.nats.client.api.StreamInfo; import io.nats.client.support.JsonUtils; -import java.time.Duration; import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.BenchmarkDriver; import io.openmessaging.benchmark.driver.BenchmarkProducer; @@ -42,8 +41,6 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.nats.client.Connection; - public class NatsBenchmarkDriver implements BenchmarkDriver { private NatsConfig config; @@ -52,24 +49,28 @@ public class NatsBenchmarkDriver implements BenchmarkDriver { private JetStream jetStream; @Override - public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException, InterruptedException { + public void initialize(File configurationFile, StatsLogger statsLogger) + throws IOException, InterruptedException { config = mapper.readValue(configurationFile, NatsConfig.class); log.info("read config file," + config.toString()); - this.connection = Nats.connect(new Options.Builder() - .server(config.natsHostUrl) - .maxReconnects(5) - .errorListener(new ErrorListener() { - @Override - public void errorOccurred(Connection conn, String error) { - log.error("Error on connection {}: {}", conn, error); - } - - @Override - public void exceptionOccurred(Connection conn, Exception exp) { - log.error("Exception on connection {}", conn, exp); - } - }) - .build()); + this.connection = + Nats.connect( + new Options.Builder() + .server(config.natsHostUrl) + .maxReconnects(5) + .errorListener( + new ErrorListener() { + @Override + public void errorOccurred(Connection conn, String error) { + log.error("Error on connection {}: {}", conn, error); + } + + @Override + public void exceptionOccurred(Connection conn, Exception exp) { + log.error("Exception on connection {}", conn, exp); + } + }) + .build()); this.jetStream = connection.jetStream(); } @@ -82,12 +83,14 @@ public String getTopicNamePrefix() { public CompletableFuture createTopic(String topic, int partitions) { try { JetStreamManagement jsm = connection.jetStreamManagement(); - StreamInfo streamInfo = jsm.addStream(StreamConfiguration.builder() - .name(topic) - .subjects(topic) - .storageType(StorageType.File) - .replicas(config.replicationFactor) - .build()); + StreamInfo streamInfo = + jsm.addStream( + StreamConfiguration.builder() + .name(topic) + .subjects(topic) + .storageType(StorageType.File) + .replicas(config.replicationFactor) + .build()); log.info("Created stream {} -- {}", topic, JsonUtils.getFormatted(streamInfo)); return CompletableFuture.completedFuture(null); } catch (Exception e) { @@ -103,18 +106,23 @@ public CompletableFuture createProducer(String topic) { } @Override - public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { - + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { Dispatcher dispatcher = connection.createDispatcher(); try { - JetStreamSubscription sub = jetStream.subscribe(topic, dispatcher, (Message msg) -> { - long publishTimestamp = readLongFromBytes(msg.getData()); - consumerCallback.messageReceived(msg.getData(), publishTimestamp); - msg.ack(); - }, false, new PushSubscribeOptions.Builder().build()); + JetStreamSubscription sub = + jetStream.subscribe( + topic, + dispatcher, + (Message msg) -> { + long publishTimestamp = readLongFromBytes(msg.getData()); + consumerCallback.messageReceived(msg.getData(), publishTimestamp); + msg.ack(); + }, + false, + new PushSubscribeOptions.Builder().build()); return CompletableFuture.completedFuture(new NatsBenchmarkConsumer()); } catch (Exception e) { CompletableFuture f = new CompletableFuture<>(); @@ -129,8 +137,9 @@ public void close() throws Exception { } private static final Logger log = LoggerFactory.getLogger(NatsBenchmarkDriver.class); - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static long readLongFromBytes(final byte[] b) { long result = 0; @@ -140,5 +149,4 @@ private static long readLongFromBytes(final byte[] b) { } return result; } - } diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java index c9f856bb3..1c9dd8a54 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkProducer.java @@ -13,19 +13,11 @@ */ package io.openmessaging.benchmark.driver.nats; + import io.nats.client.JetStream; -import io.nats.client.Message; -import io.nats.client.PublishOptions; import io.openmessaging.benchmark.driver.BenchmarkProducer; -import java.time.Duration; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import io.nats.client.Connection; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class NatsBenchmarkProducer implements BenchmarkProducer { private final String topic; @@ -44,8 +36,7 @@ public CompletableFuture sendAsync(Optional key, byte[] payload) { } @Override - public void close() throws Exception { - } + public void close() throws Exception {} public static void writeLongToBytes(long l, byte[] dst) { for (int i = 7; i >= 0; i--) { @@ -53,6 +44,4 @@ public static void writeLongToBytes(long l, byte[] dst) { l >>= 8; } } - - } diff --git a/driver-nsq/pom.xml b/driver-nsq/pom.xml index 5f16cf095..377f1f983 100644 --- a/driver-nsq/pom.xml +++ b/driver-nsq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - messaging-benchmark io.openmessaging.benchmark + messaging-benchmark 0.0.1-SNAPSHOT - 4.0.0 driver-nsq - - com.github.brainlag - nsq-client - 1.0.0.RC4 - ${project.groupId} driver-api ${project.version} + + com.github.brainlag + nsq-client + 1.0.0.RC4 + - \ No newline at end of file + diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java index a301da2ca..3e63c5e2a 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkConsumer.java @@ -13,15 +13,19 @@ */ package io.openmessaging.benchmark.driver.nsq; + import com.github.brainlag.nsq.NSQConsumer; import io.openmessaging.benchmark.driver.BenchmarkConsumer; public class NsqBenchmarkConsumer implements BenchmarkConsumer { private NSQConsumer nsqConsumer; + public NsqBenchmarkConsumer(NSQConsumer nsqConsumer) { this.nsqConsumer = nsqConsumer; } - @Override public void close() throws Exception { + + @Override + public void close() throws Exception { this.nsqConsumer.shutdown(); } } diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java index 65043b057..99f9f7586 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkDriver.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.nsq; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -23,7 +24,6 @@ import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.BenchmarkDriver; import io.openmessaging.benchmark.driver.BenchmarkProducer; - import io.openmessaging.benchmark.driver.ConsumerCallback; import java.io.File; import java.io.IOException; @@ -35,16 +35,19 @@ public class NsqBenchmarkDriver implements BenchmarkDriver { private NsqConfig config; - @Override public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException { + @Override + public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException { config = mapper.readValue(configurationFile, NsqConfig.class); log.info("read config file," + config.toString()); } - @Override public String getTopicNamePrefix() { + @Override + public String getTopicNamePrefix() { return "Nsq-Benchmark"; } - @Override public CompletableFuture createTopic(String topic, int partitions) { + @Override + public CompletableFuture createTopic(String topic, int partitions) { log.info("create a topic" + topic); log.info("ignore partitions"); CompletableFuture future = new CompletableFuture<>(); @@ -52,7 +55,8 @@ public class NsqBenchmarkDriver implements BenchmarkDriver { return future; } - @Override public CompletableFuture createProducer(final String topic) { + @Override + public CompletableFuture createProducer(final String topic) { NSQProducer nsqProducer = new NSQProducer(); nsqProducer.addAddress(config.nsqdHost, 4150); nsqProducer.start(); @@ -61,19 +65,26 @@ public class NsqBenchmarkDriver implements BenchmarkDriver { return CompletableFuture.completedFuture(new NsqBenchmarkProducer(nsqProducer, topic)); } - @Override public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { - //Channel can be treat as subscriptionName + @Override + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { + // Channel can be treat as subscriptionName NSQLookup lookup = new DefaultNSQLookup(); lookup.addLookupAddress(config.lookupHost, 4161); - NSQConsumer nsqConsumer = new NSQConsumer(lookup, topic, subscriptionName, (message) -> { - //now mark the message as finished. - consumerCallback.messageReceived(message.getMessage(), message.getTimestamp().getTime()); - message.finished(); + NSQConsumer nsqConsumer = + new NSQConsumer( + lookup, + topic, + subscriptionName, + (message) -> { + // now mark the message as finished. + consumerCallback.messageReceived( + message.getMessage(), message.getTimestamp().getTime()); + message.finished(); - //or you could requeue it, which indicates a failure and puts it back on the queue. - //message.requeue(); - }); + // or you could requeue it, which indicates a failure and puts it back on the queue. + // message.requeue(); + }); nsqConsumer.start(); log.info("start a nsq consumer"); @@ -81,11 +92,11 @@ public class NsqBenchmarkDriver implements BenchmarkDriver { return CompletableFuture.completedFuture(new NsqBenchmarkConsumer(nsqConsumer)); } - @Override public void close() throws Exception { + @Override + public void close() throws Exception {} - } private static final Logger log = LoggerFactory.getLogger(NsqBenchmarkDriver.class); - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } diff --git a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java index 1ae87645e..f1f85c5e3 100644 --- a/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java +++ b/driver-nsq/src/main/java/io/openmessaging/benchmark/driver/nsq/NsqBenchmarkProducer.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.nsq; + import com.github.brainlag.nsq.NSQProducer; import com.github.brainlag.nsq.exceptions.NSQException; import io.openmessaging.benchmark.driver.BenchmarkProducer; @@ -36,26 +37,27 @@ public NsqBenchmarkProducer(final NSQProducer nsqProducer, final String topic) { this.topic = topic; } - @Override public CompletableFuture sendAsync(Optional key, byte[] payload) { + @Override + public CompletableFuture sendAsync(Optional key, byte[] payload) { CompletableFuture future = new CompletableFuture<>(); try { semaphore.acquire(); - executor.submit(() -> { - try { - nsqProducer.produce(topic, payload); - } catch (NSQException e) { - log.error("send exception", e); - future.exceptionally(null); - } catch (TimeoutException e) { - log.error("send exception", e); - future.exceptionally(null); - } finally { - semaphore.release(); - } - future.complete(null); - - }); + executor.submit( + () -> { + try { + nsqProducer.produce(topic, payload); + } catch (NSQException e) { + log.error("send exception", e); + future.exceptionally(null); + } catch (TimeoutException e) { + log.error("send exception", e); + future.exceptionally(null); + } finally { + semaphore.release(); + } + future.complete(null); + }); } catch (InterruptedException e) { log.error("semaphore exception", e); future.exceptionally(null); @@ -64,8 +66,10 @@ public NsqBenchmarkProducer(final NSQProducer nsqProducer, final String topic) { return future; } - @Override public void close() throws Exception { + @Override + public void close() throws Exception { this.nsqProducer.shutdown(); } + private static final Logger log = LoggerFactory.getLogger(NsqBenchmarkProducer.class); } diff --git a/driver-pravega/README.md b/driver-pravega/README.md index 707f025db..e67d9ba21 100644 --- a/driver-pravega/README.md +++ b/driver-pravega/README.md @@ -1,25 +1,31 @@ # PRAVEGA BENCHMARKS -This tutorial shows you how to run OpenMessaging benchmarks for [Pravega](https://pravega.io/). +This tutorial shows you how to run OpenMessaging benchmarks for [Pravega](https://pravega.io/). You can currently deploy to the following platforms: * [Amazon Web Services (AWS)](#deploy-a-pravega-cluster-on-amazon-web-services) # INITIAL SETUP + To begin with, you will need to clone the benchmark repo from the Pravega organization on GitHub: + ``` $ git clone https://github.com/openmessaging/openmessaging-benchmark.git $ cd openmessaging-benchmark ``` + You will also need to have [Maven](https://maven.apache.org/install.html) installed. # CREATE LOCAL ARTIFACTS -Once you have the repo cloned locally, you can create all the artifacts necessary to run the benchmarks with a single + +Once you have the repo cloned locally, you can create all the artifacts necessary to run the benchmarks with a single Maven command: + ``` $ mvn install ``` -If you want to use the pre-release version of Pravega or the master branch of Pravega, please + +If you want to use the pre-release version of Pravega or the master branch of Pravega, please check [how to build Pravega](doc/build_pravega.md). # DEPLOY A PRAVEGA CLUSTER ON AMAZON WEB SERVICES @@ -28,6 +34,7 @@ You can deploy a Pravega cluster on AWS (for benchmarking purposes) using [Terra You’ll need to have both of those tools installed as well as the `terraform-inventory` [plugin](https://github.com/adammck/terraform-inventory) for Terraform. You also need to install an Ansible modules to support metrics. + ``` ansible-galaxy install cloudalchemy.node-exporter ``` @@ -38,53 +45,58 @@ In addition, you’ll need to: * [Install the `aws` CLI tool](https://aws.amazon.com/cli/) * [Configure the `aws` CLI tool](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) - # SSH KEYS + Once you’re all set up with AWS and have the necessary tools installed locally, you’ll need to create both a public and a private SSH key at `~/.ssh/pravega_aws` (private) and `~/.ssh/pravega_aws.pub` (public), respectively. ``` $ ssh-keygen -f ~/.ssh/pravega_aws ``` -When prompted to enter a passphrase, simply hit `Enter` twice. Then, make sure that the keys have been created: +When prompted to enter a passphrase, simply hit `Enter` twice. Then, make sure that the keys have been created: ``` $ ls ~/.ssh/pravega_aws* ``` # CREATE RESOURCES USING TERRAFORM + With SSH keys in place, you can create the necessary AWS resources using just a few Terraform commands: + ``` $ cd driver-pravega/deploy $ terraform init $ echo "yes" | terraform apply ``` + This will install the following [EC2](https://aws.amazon.com/ec2) instances (plus some other resources, such as a [Virtual Private Cloud](https://aws.amazon.com/vpc/) (VPC)): -| Resource | Description | Count | -| ----- | ----------- | ------ | -| Controller instances| The VMs on which a Pravega controller will run | 1 | -| Bookkeeper instances | The VMs on which a Bookkeeper and Segmentstore will run | 3 | -| ZooKeeper instances | The VMs on which a ZooKeeper node will run | 3 | -| Client instance | The VM from which the benchmarking suite itself will be run | 2 | +| Resource | Description | Count | +|----------------------|-------------------------------------------------------------|-------| +| Controller instances | The VMs on which a Pravega controller will run | 1 | +| Bookkeeper instances | The VMs on which a Bookkeeper and Segmentstore will run | 3 | +| ZooKeeper instances | The VMs on which a ZooKeeper node will run | 3 | +| Client instance | The VM from which the benchmarking suite itself will be run | 2 | When you run `terraform apply`, you will be prompted to type `yes`. Type `yes` to continue with the installation or anything else to quit. # VARIABLES + There’s a handful of configurable parameters related to the Terraform deployment that you can alter by modifying the defaults in the `terraform.tfvars` file. -| Variable | Description | Default | -| ----- | ----------- | ------ | -| `region` | The AWS region in which the Pravega cluster will be deployed | `us-west-2` | -| `public_key_path` | The path to the SSH public key that you’ve generated | `~/.ssh/pravega_aws.pub` | -| `ami` | The [Amazon Machine Image (AWI)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) to be used by the cluster’s machines | `ami-9fa343e7` | -| `instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (BookKeeper bookies), `m5.large`(Controller), `t3.small` (ZooKeeper), `c5.4xlarge` (benchmarking client) | +| Variable | Description | Default | +|-------------------|--------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| `region` | The AWS region in which the Pravega cluster will be deployed | `us-west-2` | +| `public_key_path` | The path to the SSH public key that you’ve generated | `~/.ssh/pravega_aws.pub` | +| `ami` | The [Amazon Machine Image (AWI)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) to be used by the cluster’s machines | `ami-9fa343e7` | +| `instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (BookKeeper bookies), `m5.large`(Controller), `t3.small` (ZooKeeper), `c5.4xlarge` (benchmarking client) | If you modify the `public_key_path`, make sure that you point to the appropriate SSH key path when running the [Ansible playbook](#_RUNNING_THE_ANSIBLE_PLAYBOOK). # RUNNING THE ANSIBLE PLAYBOOK With the appropriate infrastructure in place, you can install and start the Pravega cluster using Ansible with just one command: + ``` # Fixes "terraform-inventory had an execution error: Error reading tfstate file: 0.12 format error" $ export TF_STATE=./ @@ -93,14 +105,19 @@ $ ansible-playbook \ --inventory `which terraform-inventory` \ deploy.yaml ``` + If you’re using an SSH private key path different from `~/.ssh/pravega_aws`, you can specify that path using the `--private-key` flag, for example `--private-key=~/.ssh/my_key`. # SSHING INTO THE CLIENT HOST + In the [output](https://learn.hashicorp.com/terraform/getting-started/outputs.html) produced by Terraform, there’s a `client_ssh_host` variable that provides the IP address for the client EC2 host from which benchmarks can be run. You can SSH into that host using this command: + ``` $ ssh -i ~/.ssh/pravega_aws ec2-user@$(terraform output client_ssh_host) ``` + # RUNNING THE BENCHMARKS FROM THE CLIENT HOSTS + > The benchmark scripts can be run from the /opt/benchmark working directory. Once you’ve successfully SSHed into the client host, you can run any of the [existing benchmarking workloads](http://openmessaging.cloud/docs/benchmarks/#benchmarking-workloads) by specifying the YAML file for that workload when running the `benchmark` executable. All workloads are in the `workloads` folder. Here’s an example: @@ -110,24 +127,26 @@ $ sudo bin/benchmark \ --drivers driver-pravega/pravega.yaml \ workloads/1-topic-16-partitions-1kb.yaml ``` + > Although benchmarks are run from a specific client host, the benchmarks are run in distributed mode, across multiple client hosts. There are multiple Pravega “modes” for which you can run benchmarks. Each mode has its own YAML configuration file in the driver-pravega folder. -| Mode | Description | Config file | -| ----- | ----------- | ------ | -| Standard | Pravega with transaction disabled (at-least-once semantics) | [pravega.yaml](./pravega.yaml) | -| Exactly Once | Pravega with transaction enabled (exactly-once semantics) | [pravega-exactly-once.yaml](./pravega-exactly-once.yaml) | +| Mode | Description | Config file | +|--------------|-------------------------------------------------------------|----------------------------------------------------------| +| Standard | Pravega with transaction disabled (at-least-once semantics) | [pravega.yaml](./pravega.yaml) | +| Exactly Once | Pravega with transaction enabled (exactly-once semantics) | [pravega-exactly-once.yaml](./pravega-exactly-once.yaml) | The example used the “standard” mode as configured in `driver-pravega/pravega.yaml`. Here’s an example of running a benchmarking workload in exactly-once mode: + ``` $ sudo bin/benchmark \ --drivers driver-pravega/pravega-exactly-once.yaml \ workloads/1-topic-16-partitions-1kb.yaml ``` - # SPECIFY CLIENT HOSTS + By default, benchmarks will be run from the set of hosts created by Terraform. You can also specify a comma-separated list of client hosts using the `--workers` flag (or `-w` for short): ``` @@ -136,23 +155,31 @@ $ sudo bin/benchmark \ --workers 1.2.3.4:8080,4.5.6.7:8080 \ # or -w 1.2.3.4:8080,4.5.6.7:8080 workloads/1-topic-16-partitions-1kb.yaml ``` + # DOWNLOADING YOUR BENCHMARKING RESULTS + The OpenMessaging benchmarking suite stores results in JSON files in the `/opt/benchmark` folder on the client host from which the benchmarks are run. You can download those results files onto your local machine using `scp`. You can download all generated JSON results files using this command: ``` $ scp -i ~/.ssh/pravega_aws ec2-user@$(terraform output client_ssh_host):/opt/benchmark/*.json . ``` + # COLLECTING METRICS AND LOGS + See [metrics and logs](doc/metrics_and_logs.md). # TEARING DOWN YOUR BENCHMARKING INFRASTRUCTURE + Once you’re finished running your benchmarks, you should tear down the AWS infrastructure you deployed for the sake of saving costs. You can do that with one command: + ``` $ terraform destroy -force ``` + Make sure to let the process run to completion (it could take several minutes). Once the tear down is complete, all AWS resources that you created for the Pravega benchmarking suite will have been removed. # RUN IN KUBERNETES + See [run in Kubernetes](doc/run_in_k8s.md). # P3 Test Driver @@ -160,4 +187,5 @@ See [run in Kubernetes](doc/run_in_k8s.md). [P3 Test Driver](https://github.com/pravega/p3_test_driver) can be used to run multiple tests automatically and plot the results. # TROUBLESHOOTING + See [troubleshooting](doc/troubleshooting.md). diff --git a/driver-pravega/doc/build_pravega.md b/driver-pravega/doc/build_pravega.md index b1a82a20c..1808b7690 100644 --- a/driver-pravega/doc/build_pravega.md +++ b/driver-pravega/doc/build_pravega.md @@ -11,6 +11,7 @@ git checkout master This will build the file `pravega/build/distributions/pravega-0.9.0.tgz.` Then comment `pravegaSrc` and `pravegaSrcRemote: yes` and uncomment `pravegaSrc` `pravegaSrcRemote: no` in `driver-pravega/deploy/deploy.yaml` + ``` # Change below to use a published release of Pravega or a local build. # pravegaSrc: "https://github.com/pravega/pravega/releases/download/v{{ pravegaVersion }}/pravega-{{ pravegaVersion }}.tgz" @@ -19,12 +20,16 @@ Then comment `pravegaSrc` and `pravegaSrcRemote: yes` and uncomment `pravegaSrc` pravegaSrc: "../../../pravega/build/distributions/pravega-{{ pravegaVersion }}.tgz" pravegaSrcRemote: no ``` + If needed, change the variable `pravegaVersion` in [vars.yaml](../deploy/vars.yaml) to match the version built. If needed, change [pom.xml](../pom.xml) to match the version built. ## Build Benchmark + Add flag to skip license check`-Dlicense.skip=true` if license check failed. + ``` mvn clean install ``` + diff --git a/driver-pravega/doc/metrics_and_logs.md b/driver-pravega/doc/metrics_and_logs.md index 929492273..9e72570eb 100644 --- a/driver-pravega/doc/metrics_and_logs.md +++ b/driver-pravega/doc/metrics_and_logs.md @@ -44,12 +44,12 @@ Login using user name "admin" and any password. Configure Grafana with the following data sources: - - Prometheus - - Name: Prometheus - - HTTP URL: http://prometheus:9090 - - InfluxDB - - Name: pravega-influxdb - - HTTP URL: http://influxdb:8086 - - InfluxDB Details Database: pravega +- Prometheus + - Name: Prometheus + - HTTP URL: http://prometheus:9090 +- InfluxDB + - Name: pravega-influxdb + - HTTP URL: http://influxdb:8086 + - InfluxDB Details Database: pravega Load dashboards from [deploy/templates/dashboards](../deploy/templates/dashboards). diff --git a/driver-pravega/doc/run_in_k8s.md b/driver-pravega/doc/run_in_k8s.md index 73ee6aa5f..073bc2f89 100644 --- a/driver-pravega/doc/run_in_k8s.md +++ b/driver-pravega/doc/run_in_k8s.md @@ -7,6 +7,7 @@ ``` ## Run local driver on Kubernetes: + ``` kubectl run -n examples --rm -it --image pravega/openmessaging-benchmark:latest --serviceaccount examples-pravega openmessaging-benchmark ``` @@ -15,3 +16,5 @@ kubectl run -n examples --rm -it --image pravega/openmessaging-benchmark:latest ``` ./deploy-k8s-components.sh +``` + diff --git a/driver-pravega/doc/troubleshooting.md b/driver-pravega/doc/troubleshooting.md index 6ff3c0927..2ec033937 100644 --- a/driver-pravega/doc/troubleshooting.md +++ b/driver-pravega/doc/troubleshooting.md @@ -13,3 +13,4 @@ journalctl -u pravega-segmentstore ``` export TF_STATE=./ ``` + diff --git a/driver-pravega/pom.xml b/driver-pravega/pom.xml index 6c8071f3a..5cce48a41 100644 --- a/driver-pravega/pom.xml +++ b/driver-pravega/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - driver-pulsar - - 2.10.1 - + driver-pulsar + + 2.10.1 + - - - ${project.groupId} - driver-api - ${project.version} - - - org.apache.pulsar - pulsar-client-all - ${pulsar.version} - - - com.google.guava - guava - - + + + ${project.groupId} + driver-api + ${project.version} + + + com.google.guava + guava + + + org.apache.pulsar + pulsar-client-all + ${pulsar.version} + + diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java index 84e9305e0..064cf8e98 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkConsumer.java @@ -14,9 +14,9 @@ package io.openmessaging.benchmark.driver.pulsar; import static java.util.Collections.unmodifiableList; + import io.openmessaging.benchmark.driver.BenchmarkConsumer; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.List; import org.apache.pulsar.client.api.Consumer; diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java index 08778fc63..dd02fb874 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java @@ -13,6 +13,19 @@ */ package io.openmessaging.benchmark.driver.pulsar; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.common.collect.Sets; +import com.google.common.io.BaseEncoding; +import io.openmessaging.benchmark.driver.BenchmarkConsumer; +import io.openmessaging.benchmark.driver.BenchmarkDriver; +import io.openmessaging.benchmark.driver.BenchmarkProducer; +import io.openmessaging.benchmark.driver.ConsumerCallback; +import io.openmessaging.benchmark.driver.pulsar.config.PulsarClientConfig.PersistenceConfiguration; +import io.openmessaging.benchmark.driver.pulsar.config.PulsarConfig; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -22,7 +35,6 @@ import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; - import java.util.stream.Collectors; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -44,20 +56,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.common.collect.Sets; -import com.google.common.io.BaseEncoding; - -import io.openmessaging.benchmark.driver.BenchmarkConsumer; -import io.openmessaging.benchmark.driver.BenchmarkDriver; -import io.openmessaging.benchmark.driver.BenchmarkProducer; -import io.openmessaging.benchmark.driver.ConsumerCallback; -import io.openmessaging.benchmark.driver.pulsar.config.PulsarClientConfig.PersistenceConfiguration; -import io.openmessaging.benchmark.driver.pulsar.config.PulsarConfig; - public class PulsarBenchmarkDriver implements BenchmarkDriver { private PulsarClient client; @@ -73,33 +71,40 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I this.config = readConfig(configurationFile); log.info("Pulsar driver configuration: {}", writer.writeValueAsString(config)); - ClientBuilder clientBuilder = PulsarClient.builder() - .ioThreads(config.client.ioThreads) - .connectionsPerBroker(config.client.connectionsPerBroker) - .statsInterval(0, TimeUnit.SECONDS) - .serviceUrl(config.client.serviceUrl) - .maxConcurrentLookupRequests(config.client.maxConcurrentLookupRequests) - .maxLookupRequests(Integer.MAX_VALUE) - .memoryLimit(config.client.clientMemoryLimitMB, SizeUnit.MEGA_BYTES) - .operationTimeout(10, TimeUnit.MINUTES) - .listenerThreads(Runtime.getRuntime().availableProcessors()); + ClientBuilder clientBuilder = + PulsarClient.builder() + .ioThreads(config.client.ioThreads) + .connectionsPerBroker(config.client.connectionsPerBroker) + .statsInterval(0, TimeUnit.SECONDS) + .serviceUrl(config.client.serviceUrl) + .maxConcurrentLookupRequests(config.client.maxConcurrentLookupRequests) + .maxLookupRequests(Integer.MAX_VALUE) + .memoryLimit(config.client.clientMemoryLimitMB, SizeUnit.MEGA_BYTES) + .operationTimeout(10, TimeUnit.MINUTES) + .listenerThreads(Runtime.getRuntime().availableProcessors()); if (config.client.serviceUrl.startsWith("pulsar+ssl")) { - clientBuilder.allowTlsInsecureConnection(config.client.tlsAllowInsecureConnection) - .enableTlsHostnameVerification(config.client.tlsEnableHostnameVerification) - .tlsTrustCertsFilePath(config.client.tlsTrustCertsFilePath); + clientBuilder + .allowTlsInsecureConnection(config.client.tlsAllowInsecureConnection) + .enableTlsHostnameVerification(config.client.tlsEnableHostnameVerification) + .tlsTrustCertsFilePath(config.client.tlsTrustCertsFilePath); } - PulsarAdminBuilder pulsarAdminBuilder = PulsarAdmin.builder().serviceHttpUrl(config.client.httpUrl); + PulsarAdminBuilder pulsarAdminBuilder = + PulsarAdmin.builder().serviceHttpUrl(config.client.httpUrl); if (config.client.httpUrl.startsWith("https")) { - pulsarAdminBuilder.allowTlsInsecureConnection(config.client.tlsAllowInsecureConnection) - .enableTlsHostnameVerification(config.client.tlsEnableHostnameVerification) - .tlsTrustCertsFilePath(config.client.tlsTrustCertsFilePath); + pulsarAdminBuilder + .allowTlsInsecureConnection(config.client.tlsAllowInsecureConnection) + .enableTlsHostnameVerification(config.client.tlsEnableHostnameVerification) + .tlsTrustCertsFilePath(config.client.tlsTrustCertsFilePath); } - if (config.client.authentication.plugin != null && !config.client.authentication.plugin.isEmpty()) { - clientBuilder.authentication(config.client.authentication.plugin, config.client.authentication.data); - pulsarAdminBuilder.authentication(config.client.authentication.plugin, config.client.authentication.data); + if (config.client.authentication.plugin != null + && !config.client.authentication.plugin.isEmpty()) { + clientBuilder.authentication( + config.client.authentication.plugin, config.client.authentication.data); + pulsarAdminBuilder.authentication( + config.client.authentication.plugin, config.client.authentication.data); } client = clientBuilder.build(); @@ -110,14 +115,17 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I log.info("Created Pulsar admin client for HTTP URL {}", config.client.httpUrl); - producerBuilder = client.newProducer() - .enableBatching(config.producer.batchingEnabled) - .batchingMaxPublishDelay(config.producer.batchingMaxPublishDelayMs, TimeUnit.MILLISECONDS) - .batchingMaxMessages(Integer.MAX_VALUE) - .batchingMaxBytes(config.producer.batchingMaxBytes) - .blockIfQueueFull(config.producer.blockIfQueueFull) - .sendTimeout(0, TimeUnit.MILLISECONDS) - .maxPendingMessages(config.producer.pendingQueueSize); + producerBuilder = + client + .newProducer() + .enableBatching(config.producer.batchingEnabled) + .batchingMaxPublishDelay( + config.producer.batchingMaxPublishDelayMs, TimeUnit.MILLISECONDS) + .batchingMaxMessages(Integer.MAX_VALUE) + .batchingMaxBytes(config.producer.batchingMaxBytes) + .blockIfQueueFull(config.producer.blockIfQueueFull) + .sendTimeout(0, TimeUnit.MILLISECONDS) + .maxPendingMessages(config.producer.pendingQueueSize); try { // Create namespace and set the configuration @@ -125,8 +133,14 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I String cluster = config.client.clusterName; if (!adminClient.tenants().getTenants().contains(tenant)) { try { - adminClient.tenants().createTenant(tenant, - TenantInfo.builder().adminRoles(Collections.emptySet()).allowedClusters(Sets.newHashSet(cluster)).build()); + adminClient + .tenants() + .createTenant( + tenant, + TenantInfo.builder() + .adminRoles(Collections.emptySet()) + .allowedClusters(Sets.newHashSet(cluster)) + .build()); } catch (ConflictException e) { // Ignore. This can happen when multiple workers are initializing at the same time } @@ -138,18 +152,27 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I log.info("Created Pulsar namespace {}", namespace); PersistenceConfiguration p = config.client.persistence; - adminClient.namespaces().setPersistence(namespace, - new PersistencePolicies(p.ensembleSize, p.writeQuorum, p.ackQuorum, 1.0)); - - adminClient.namespaces().setBacklogQuota(namespace, - BacklogQuota.builder() - .limitSize(-1L) - .limitTime(-1) - .retentionPolicy(RetentionPolicy.producer_exception) - .build()); + adminClient + .namespaces() + .setPersistence( + namespace, new PersistencePolicies(p.ensembleSize, p.writeQuorum, p.ackQuorum, 1.0)); + + adminClient + .namespaces() + .setBacklogQuota( + namespace, + BacklogQuota.builder() + .limitSize(-1L) + .limitTime(-1) + .retentionPolicy(RetentionPolicy.producer_exception) + .build()); adminClient.namespaces().setDeduplicationStatus(namespace, p.deduplicationEnabled); - log.info("Applied persistence configuration for namespace {}/{}/{}: {}", tenant, cluster, namespace, - writer.writeValueAsString(p)); + log.info( + "Applied persistence configuration for namespace {}/{}/{}: {}", + tenant, + cluster, + namespace, + writer.writeValueAsString(p)); } catch (PulsarAdminException e) { throw new IOException(e); @@ -173,37 +196,42 @@ public CompletableFuture createTopic(String topic, int partitions) { @Override public CompletableFuture createProducer(String topic) { - return producerBuilder.topic(topic).createAsync() - .thenApply(PulsarBenchmarkProducer::new); + return producerBuilder.topic(topic).createAsync().thenApply(PulsarBenchmarkProducer::new); } @Override - public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { List>> futures = new ArrayList<>(); - return client.getPartitionsForTopic(topic) - .thenCompose(partitions -> { - partitions.forEach(p -> futures.add(createInternalConsumer(p, subscriptionName, consumerCallback))); - return FutureUtil.waitForAll(futures); - }).thenApply(__ -> new PulsarBenchmarkConsumer( - futures.stream().map(CompletableFuture::join).collect(Collectors.toList()) - ) - ); + return client + .getPartitionsForTopic(topic) + .thenCompose( + partitions -> { + partitions.forEach( + p -> futures.add(createInternalConsumer(p, subscriptionName, consumerCallback))); + return FutureUtil.waitForAll(futures); + }) + .thenApply( + __ -> + new PulsarBenchmarkConsumer( + futures.stream().map(CompletableFuture::join).collect(Collectors.toList()))); } - CompletableFuture> createInternalConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { - return client.newConsumer(Schema.BYTEBUFFER) + CompletableFuture> createInternalConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { + return client + .newConsumer(Schema.BYTEBUFFER) .priorityLevel(0) .subscriptionType(SubscriptionType.Failover) - .messageListener((c, msg) -> { - try { - consumerCallback.messageReceived(msg.getValue(), msg.getPublishTime()); - c.acknowledgeAsync(msg); - } finally { - msg.release(); - } - }) + .messageListener( + (c, msg) -> { + try { + consumerCallback.messageReceived(msg.getValue(), msg.getPublishTime()); + c.acknowledgeAsync(msg); + } finally { + msg.release(); + } + }) .topic(topic) .subscriptionName(subscriptionName) .receiverQueueSize(config.consumer.receiverQueueSize) @@ -227,7 +255,8 @@ public void close() throws Exception { log.info("Pulsar benchmark driver successfully shut down"); } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static PulsarConfig readConfig(File configurationFile) throws IOException { @@ -236,7 +265,7 @@ private static PulsarConfig readConfig(File configurationFile) throws IOExceptio private static final Random random = new Random(); - private static final String getRandomString() { + private static String getRandomString() { byte[] buffer = new byte[5]; random.nextBytes(buffer); return BaseEncoding.base64Url().omitPadding().encode(buffer); diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java index cad7a4a12..80adb2add 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkProducer.java @@ -13,14 +13,13 @@ */ package io.openmessaging.benchmark.driver.pulsar; + +import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.Optional; import java.util.concurrent.CompletableFuture; - import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.TypedMessageBuilder; -import io.openmessaging.benchmark.driver.BenchmarkProducer; - public class PulsarBenchmarkProducer implements BenchmarkProducer { private final Producer producer; @@ -43,5 +42,4 @@ public CompletableFuture sendAsync(Optional key, byte[] payload) { return msgBuilder.sendAsync().thenApply(msgId -> null); } - } diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java index 15e350b4e..0dedaa5d2 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarClientConfig.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.pulsar.config; + import org.apache.pulsar.common.naming.TopicDomain; public class PulsarClientConfig { diff --git a/driver-rabbitmq/README.md b/driver-rabbitmq/README.md index 16ef49782..fc8b68d69 100644 --- a/driver-rabbitmq/README.md +++ b/driver-rabbitmq/README.md @@ -53,10 +53,10 @@ $ terraform apply That will install the following [EC2](https://aws.amazon.com/ec2) instances (plus some other resources, such as a [Virtual Private Cloud](https://aws.amazon.com/vpc/) (VPC)): -Resource | Description | Count -:--------|:------------|:----- -RabbitMQ instances | The VMs on which RabbitMQ brokers will run | 3 -Client instance | The VM from which the benchmarking suite itself will be run | 1 +| Resource | Description | Count | +|:-------------------|:------------------------------------------------------------|:------| +| RabbitMQ instances | The VMs on which RabbitMQ brokers will run | 3 | +| Client instance | The VM from which the benchmarking suite itself will be run | 1 | When you run `terraform apply`, you will be prompted to type `yes`. Type `yes` to continue with the installation or anything else to quit. @@ -66,20 +66,20 @@ Once the installation is complete, you will see a confirmation message listing t There's a handful of configurable parameters related to the Terraform deployment that you can alter by modifying the defaults in the `terraform.tfvars` file. -Variable | Description | Default -:--------|:------------|:------- -`region` | The AWS region in which the RabbitMQ cluster will be deployed | `us-west-2` -`az` | The availability zone in which the RabbitMQ cluster will be deployed | `us-west-2a` -`public_key_path` | The path to the SSH public key that you've generated | `~/.ssh/rabbitmq_aws.pub` -`ami` | The [Amazon Machine Image](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) (AWI) to be used by the cluster's machines | [`ami-9fa343e7`](https://access.redhat.com/articles/3135091) -`instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (RabbitMQ brokers), `c4.8xlarge` (benchmarking client) +| Variable | Description | Default | +|:------------------|:------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------| +| `region` | The AWS region in which the RabbitMQ cluster will be deployed | `us-west-2` | +| `az` | The availability zone in which the RabbitMQ cluster will be deployed | `us-west-2a` | +| `public_key_path` | The path to the SSH public key that you've generated | `~/.ssh/rabbitmq_aws.pub` | +| `ami` | The [Amazon Machine Image](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) (AWI) to be used by the cluster's machines | [`ami-9fa343e7`](https://access.redhat.com/articles/3135091) | +| `instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (RabbitMQ brokers), `c4.8xlarge` (benchmarking client) | > If you modify the `public_key_path`, make sure that you point to the appropriate SSH key path when running the [Ansible playbook](#running-the-ansible-playbook). ### Running the Ansible playbook With the appropriate infrastructure in place, you can install and start the RabbitMQ cluster using Ansible with just one command. -Note that a `TFSTATE` environment must point to the folder in which the `tf.state` file is located. +Note that a `TFSTATE` environment must point to the folder in which the `tf.state` file is located. ```bash $ TF_STATE=. ansible-playbook \ @@ -120,4 +120,5 @@ The `rabbitmq_management` plugin is installed, and the HTTP endpoint is exposed to access the management REST API at this endpoint. Note that the connection is authenticated but not currently encrypted and so passwords will be passed in plain text. Use -the `admin` account configured in the [Terraform](deploy/provision-rabbitmq-aws.tf) file to log in. \ No newline at end of file +the `admin` account configured in the [Terraform](deploy/provision-rabbitmq-aws.tf) file to log in. + diff --git a/driver-rabbitmq/pom.xml b/driver-rabbitmq/pom.xml index 04b95ad32..f6fe0fe29 100644 --- a/driver-rabbitmq/pom.xml +++ b/driver-rabbitmq/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - - driver-rabbitmq + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + + driver-rabbitmq - - - ${project.groupId} - driver-api - ${project.version} - - - com.rabbitmq - amqp-client - 4.8.0 - - - io.netty - netty-all - - + + + ${project.groupId} + driver-api + ${project.version} + + + com.rabbitmq + amqp-client + 4.8.0 + + + io.netty + netty-all + + diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java index 975121e18..27f87d36e 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkConsumer.java @@ -13,16 +13,15 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; -import com.rabbitmq.client.AlreadyClosedException; -import java.io.IOException; import com.rabbitmq.client.AMQP.BasicProperties; +import com.rabbitmq.client.AlreadyClosedException; import com.rabbitmq.client.Channel; import com.rabbitmq.client.DefaultConsumer; import com.rabbitmq.client.Envelope; - import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.ConsumerCallback; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +32,8 @@ public class RabbitMqBenchmarkConsumer extends DefaultConsumer implements Benchm private final Channel channel; private final ConsumerCallback callback; - public RabbitMqBenchmarkConsumer(Channel channel, String queueName, ConsumerCallback callback) throws IOException { + public RabbitMqBenchmarkConsumer(Channel channel, String queueName, ConsumerCallback callback) + throws IOException { super(channel); this.channel = channel; @@ -42,7 +42,8 @@ public RabbitMqBenchmarkConsumer(Channel channel, String queueName, ConsumerCall } @Override - public void handleDelivery(String consumerTag, Envelope envelope, BasicProperties properties, byte[] body) { + public void handleDelivery( + String consumerTag, Envelope envelope, BasicProperties properties, byte[] body) { callback.messageReceived(body, properties.getTimestamp().getTime()); } @@ -54,5 +55,4 @@ public void close() throws Exception { log.warn("Channel already closed", e); } } - } diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index 6181d15c5..d56442205 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -13,32 +13,31 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; -import com.rabbitmq.client.AlreadyClosedException; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.atomic.AtomicInteger; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.rabbitmq.client.AlreadyClosedException; import com.rabbitmq.client.BuiltinExchangeType; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; - import io.netty.handler.codec.http.QueryStringDecoder; import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.BenchmarkDriver; import io.openmessaging.benchmark.driver.BenchmarkProducer; import io.openmessaging.benchmark.driver.ConsumerCallback; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +55,8 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I @Override public void close() { - for(Iterator> it = connections.entrySet().iterator(); it.hasNext(); ) { + for (Iterator> it = connections.entrySet().iterator(); + it.hasNext(); ) { Connection connection = it.next().getValue(); try { connection.close(); @@ -67,13 +67,13 @@ public void close() { } it.remove(); } - } @Override public String getTopicNamePrefix() { // Do a round-robin on AMQP URIs - URI configUri = URI.create(config.amqpUris.get(uriIndex.getAndIncrement() % config.amqpUris.size())); + URI configUri = + URI.create(config.amqpUris.get(uriIndex.getAndIncrement() % config.amqpUris.size())); URI topicUri = configUri.resolve(configUri.getRawPath() + "?exchange=test-exchange"); return topicUri.toString(); } @@ -93,43 +93,54 @@ public CompletableFuture createTopic(String topic, int partitions) { @Override public CompletableFuture createProducer(String topic) { CompletableFuture future = new CompletableFuture<>(); - ForkJoinPool.commonPool().execute(() -> { - try { - String uri = topic.split("\\?")[0]; - Connection connection = getOrCreateConnection(uri); - Channel channel = connection.createChannel(); - String exchange = getExchangeName(topic); - channel.exchangeDeclare(exchange, BuiltinExchangeType.FANOUT, true); - channel.confirmSelect(); - future.complete(new RabbitMqBenchmarkProducer(channel, exchange, config.messagePersistence)); - } catch (Exception e) { - future.completeExceptionally(e); - } - }); + ForkJoinPool.commonPool() + .execute( + () -> { + try { + String uri = topic.split("\\?")[0]; + Connection connection = getOrCreateConnection(uri); + Channel channel = connection.createChannel(); + String exchange = getExchangeName(topic); + channel.exchangeDeclare(exchange, BuiltinExchangeType.FANOUT, true); + channel.confirmSelect(); + future.complete( + new RabbitMqBenchmarkProducer(channel, exchange, config.messagePersistence)); + } catch (Exception e) { + future.completeExceptionally(e); + } + }); return future; } @Override - public CompletableFuture createConsumer(String topic, String subscriptionName, - ConsumerCallback consumerCallback) { + public CompletableFuture createConsumer( + String topic, String subscriptionName, ConsumerCallback consumerCallback) { CompletableFuture future = new CompletableFuture<>(); - ForkJoinPool.commonPool().execute(() -> { - try { - String uri = topic.split("\\?")[0]; - Connection connection = getOrCreateConnection(uri); - Channel channel = connection.createChannel(); - String exchange = getExchangeName(topic); - String queueName = exchange + "-" + subscriptionName; - channel.exchangeDeclare(exchange, BuiltinExchangeType.FANOUT, true); - // Create the queue - channel.queueDeclare(queueName, true, false, false, Collections.singletonMap("x-queue-type", "quorum")); - channel.queueBind(queueName, exchange, ""); - future.complete(new RabbitMqBenchmarkConsumer(channel, queueName, consumerCallback)); - } catch (IOException e) { - future.completeExceptionally(e); - } - }); + ForkJoinPool.commonPool() + .execute( + () -> { + try { + String uri = topic.split("\\?")[0]; + Connection connection = getOrCreateConnection(uri); + Channel channel = connection.createChannel(); + String exchange = getExchangeName(topic); + String queueName = exchange + "-" + subscriptionName; + channel.exchangeDeclare(exchange, BuiltinExchangeType.FANOUT, true); + // Create the queue + channel.queueDeclare( + queueName, + true, + false, + false, + Collections.singletonMap("x-queue-type", "quorum")); + channel.queueBind(queueName, exchange, ""); + future.complete( + new RabbitMqBenchmarkConsumer(channel, queueName, consumerCallback)); + } catch (IOException e) { + future.completeExceptionally(e); + } + }); return future; } @@ -138,27 +149,30 @@ private String getExchangeName(String uri) { QueryStringDecoder decoder = new QueryStringDecoder(uri); Map> parameters = decoder.parameters(); - if(!parameters.containsKey("exchange")) { + if (!parameters.containsKey("exchange")) { throw new IllegalArgumentException("Missing exchange param"); } return parameters.get("exchange").get(0); } private Connection getOrCreateConnection(String uri) { - return connections.computeIfAbsent(uri, uriKey -> { - try { - ConnectionFactory connectionFactory = new ConnectionFactory(); - connectionFactory.setAutomaticRecoveryEnabled(true); - connectionFactory.setUri(uri); - return connectionFactory.newConnection(); - } catch (Exception e) { - throw new RuntimeException("Couldn't establish connection", e); - } - }); + return connections.computeIfAbsent( + uri, + uriKey -> { + try { + ConnectionFactory connectionFactory = new ConnectionFactory(); + connectionFactory.setAutomaticRecoveryEnabled(true); + connectionFactory.setUri(uri); + return connectionFactory.newConnection(); + } catch (Exception e) { + throw new RuntimeException("Couldn't establish connection", e); + } + }); } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static final Logger log = LoggerFactory.getLogger(RabbitMqBenchmarkDriver.class); } diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java index f795158ee..d7d3841ad 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkProducer.java @@ -13,8 +13,12 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; + +import com.rabbitmq.client.AMQP.BasicProperties; import com.rabbitmq.client.AlreadyClosedException; +import com.rabbitmq.client.Channel; import com.rabbitmq.client.ConfirmListener; +import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.Collections; import java.util.Date; import java.util.Iterator; @@ -22,11 +26,6 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; - -import com.rabbitmq.client.AMQP.BasicProperties; -import com.rabbitmq.client.Channel; - -import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,67 +37,72 @@ public class RabbitMqBenchmarkProducer implements BenchmarkProducer { private final Channel channel; private final String exchange; private final ConfirmListener listener; - /**To record msg and it's future structure.**/ + /** To record msg and it's future structure. */ volatile SortedSet ackSet = Collections.synchronizedSortedSet(new TreeSet<>()); - private final ConcurrentHashMap> futureConcurrentHashMap = new ConcurrentHashMap<>(); + + private final ConcurrentHashMap> futureConcurrentHashMap = + new ConcurrentHashMap<>(); private final boolean messagePersistence; public RabbitMqBenchmarkProducer(Channel channel, String exchange, boolean messagePersistence) { this.channel = channel; this.exchange = exchange; this.messagePersistence = messagePersistence; - this.listener = new ConfirmListener() { - @Override - public void handleNack(long deliveryTag, boolean multiple) { - if (multiple) { - SortedSet treeHeadSet = ackSet.headSet(deliveryTag + 1); - synchronized(ackSet) { - for(Iterator iterator = treeHeadSet.iterator(); iterator.hasNext();) { - long value = iterator.next(); - iterator.remove(); - CompletableFuture future = futureConcurrentHashMap.get(value); + this.listener = + new ConfirmListener() { + @Override + public void handleNack(long deliveryTag, boolean multiple) { + if (multiple) { + SortedSet treeHeadSet = ackSet.headSet(deliveryTag + 1); + synchronized (ackSet) { + for (Iterator iterator = treeHeadSet.iterator(); iterator.hasNext(); ) { + long value = iterator.next(); + iterator.remove(); + CompletableFuture future = futureConcurrentHashMap.get(value); + if (future != null) { + future.completeExceptionally( + new RuntimeException("Message was negatively acknowledged")); + futureConcurrentHashMap.remove(value); + } + } + treeHeadSet.clear(); + } + + } else { + CompletableFuture future = futureConcurrentHashMap.get(deliveryTag); if (future != null) { - future.completeExceptionally(new RuntimeException("Message was negatively acknowledged")); - futureConcurrentHashMap.remove(value); + future.completeExceptionally( + new RuntimeException("Message was negatively acknowledged")); + futureConcurrentHashMap.remove(deliveryTag); } + ackSet.remove(deliveryTag); } - treeHeadSet.clear(); } - } else { - CompletableFuture future = futureConcurrentHashMap.get(deliveryTag); - if (future != null) { - future.completeExceptionally(new RuntimeException("Message was negatively acknowledged")); - futureConcurrentHashMap.remove(deliveryTag); - } - ackSet.remove(deliveryTag); - } - } - @Override - public void handleAck(long deliveryTag, boolean multiple) { - if (multiple) { - SortedSet treeHeadSet = ackSet.headSet(deliveryTag + 1); - synchronized(ackSet) { - for(long value : treeHeadSet) { - CompletableFuture future = futureConcurrentHashMap.get(value); + @Override + public void handleAck(long deliveryTag, boolean multiple) { + if (multiple) { + SortedSet treeHeadSet = ackSet.headSet(deliveryTag + 1); + synchronized (ackSet) { + for (long value : treeHeadSet) { + CompletableFuture future = futureConcurrentHashMap.get(value); + if (future != null) { + future.complete(null); + futureConcurrentHashMap.remove(value); + } + } + treeHeadSet.clear(); + } + } else { + CompletableFuture future = futureConcurrentHashMap.get(deliveryTag); if (future != null) { future.complete(null); - futureConcurrentHashMap.remove(value); + futureConcurrentHashMap.remove(deliveryTag); } + ackSet.remove(deliveryTag); } - treeHeadSet.clear(); - } - } else { - CompletableFuture future = futureConcurrentHashMap.get(deliveryTag); - if (future != null) { - future.complete(null); - futureConcurrentHashMap.remove(deliveryTag); } - ackSet.remove(deliveryTag); - } - - } - }; + }; channel.addConfirmListener(listener); } @@ -133,5 +137,4 @@ public CompletableFuture sendAsync(Optional key, byte[] payload) { return future; } - } diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java index 6518e44fe..4d2382046 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; + import java.util.ArrayList; import java.util.List; diff --git a/driver-redis/pom.xml b/driver-redis/pom.xml index 95e079b07..38c9a2855 100644 --- a/driver-redis/pom.xml +++ b/driver-redis/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - messaging-benchmark io.openmessaging.benchmark + messaging-benchmark 0.0.1-SNAPSHOT - 4.0.0 driver-redis @@ -31,15 +31,15 @@ driver-api ${project.version} + + com.google.guava + guava + redis.clients jedis 3.7.0 - - com.google.guava - guava - diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java index c9d8b60a2..877d12080 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java @@ -13,8 +13,16 @@ */ package io.openmessaging.benchmark.driver.redis; +import static java.nio.charset.StandardCharsets.UTF_8; + import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.ConsumerCallback; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -23,14 +31,6 @@ import redis.clients.jedis.StreamEntryID; import redis.clients.jedis.params.XReadGroupParams; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - public class RedisBenchmarkConsumer implements BenchmarkConsumer { private final JedisPool pool; private final String topic; @@ -40,7 +40,12 @@ public class RedisBenchmarkConsumer implements BenchmarkConsumer { private final Future consumerTask; private volatile boolean closing = false; - public RedisBenchmarkConsumer(final String consumerId, final String topic, final String subscriptionName, final JedisPool pool, ConsumerCallback consumerCallback) { + public RedisBenchmarkConsumer( + final String consumerId, + final String topic, + final String subscriptionName, + final JedisPool pool, + ConsumerCallback consumerCallback) { this.pool = pool; this.topic = topic; this.subscriptionName = subscriptionName; @@ -48,31 +53,34 @@ public RedisBenchmarkConsumer(final String consumerId, final String topic, final this.executor = Executors.newSingleThreadExecutor(); Jedis jedis = this.pool.getResource(); + this.consumerTask = + this.executor.submit( + () -> { + while (!closing) { + try { + Map streamQuery = + Collections.singletonMap(this.topic, StreamEntryID.UNRECEIVED_ENTRY); + List>> range = + jedis.xreadGroup( + this.subscriptionName, + this.consumerId, + XReadGroupParams.xReadGroupParams().block(0), + streamQuery); + if (range != null) { + for (Map.Entry> streamEntries : range) { + for (StreamEntry entry : streamEntries.getValue()) { + long timestamp = entry.getID().getTime(); + byte[] payload = entry.getFields().get("payload").getBytes(UTF_8); + consumerCallback.messageReceived(payload, timestamp); + } + } + } - this.consumerTask = this.executor.submit(() -> { - while (!closing) { - try { - Map streamQuery = Collections.singletonMap(this.topic, StreamEntryID.UNRECEIVED_ENTRY); - List>> range = jedis.xreadGroup(this.subscriptionName, this.consumerId, - XReadGroupParams.xReadGroupParams().block(0), streamQuery); - if (range!=null){ - for (Map.Entry> streamEntries: - range) { - for (StreamEntry entry: - streamEntries.getValue()) { - long timestamp = entry.getID().getTime(); - byte[]payload = entry.getFields().get("payload").getBytes(StandardCharsets.UTF_8); - consumerCallback.messageReceived(payload, timestamp); + } catch (Exception e) { + log.error("Failed to read from consumer instance.", e); + } } - } - } - - } catch (Exception e) { - log.error("Failed to read from consumer instance.", e); - } - } - }); - + }); } @Override @@ -82,6 +90,6 @@ public void close() throws Exception { consumerTask.get(); pool.close(); } - private static final Logger log = LoggerFactory.getLogger(RedisBenchmarkDriver.class); + private static final Logger log = LoggerFactory.getLogger(RedisBenchmarkDriver.class); } diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java index 08b1833c5..2579ccb30 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.redis; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -21,6 +22,7 @@ import io.openmessaging.benchmark.driver.BenchmarkDriver; import io.openmessaging.benchmark.driver.BenchmarkProducer; import io.openmessaging.benchmark.driver.ConsumerCallback; +import io.openmessaging.benchmark.driver.redis.client.RedisClientConfig; import java.io.File; import java.io.IOException; import java.util.Random; @@ -28,16 +30,17 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; -import io.openmessaging.benchmark.driver.redis.client.RedisClientConfig; - +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; public class RedisBenchmarkDriver implements BenchmarkDriver { JedisPool jedisPool; private RedisClientConfig clientConfig; @Override - public void initialize(final File configurationFile, final StatsLogger statsLogger) throws IOException { + public void initialize(final File configurationFile, final StatsLogger statsLogger) + throws IOException { this.clientConfig = readConfig(configurationFile); } @@ -48,8 +51,7 @@ public String getTopicNamePrefix() { @Override public CompletableFuture createTopic(final String topic, final int partitions) { - return CompletableFuture.runAsync(() -> { - }); + return CompletableFuture.runAsync(() -> {}); } @Override @@ -61,9 +63,9 @@ public CompletableFuture createProducer(final String topic) { } @Override - public CompletableFuture createConsumer(final String topic, final String subscriptionName, - final ConsumerCallback consumerCallback) { - String consumerId = "consumer-"+getRandomString(); + public CompletableFuture createConsumer( + final String topic, final String subscriptionName, final ConsumerCallback consumerCallback) { + String consumerId = "consumer-" + getRandomString(); if (jedisPool == null) { setupJedisConn(); } @@ -72,21 +74,37 @@ public CompletableFuture createConsumer(final String topic, f } catch (Exception e) { log.info("Failed to create consumer instance.", e); } - return CompletableFuture.completedFuture(new RedisBenchmarkConsumer( consumerId, topic, subscriptionName,jedisPool, consumerCallback)); + return CompletableFuture.completedFuture( + new RedisBenchmarkConsumer( + consumerId, topic, subscriptionName, jedisPool, consumerCallback)); } private void setupJedisConn() { JedisPoolConfig poolConfig = new JedisPoolConfig(); poolConfig.setMaxTotal(this.clientConfig.jedisPoolMaxTotal); poolConfig.setMaxIdle(this.clientConfig.jedisPoolMaxIdle); - if( this.clientConfig.redisPass != null ){ - if ( this.clientConfig.redisUser != null){ - jedisPool = new JedisPool(poolConfig, this.clientConfig.redisHost, this.clientConfig.redisPort, 2000, this.clientConfig.redisPass, this.clientConfig.redisUser); + if (this.clientConfig.redisPass != null) { + if (this.clientConfig.redisUser != null) { + jedisPool = + new JedisPool( + poolConfig, + this.clientConfig.redisHost, + this.clientConfig.redisPort, + 2000, + this.clientConfig.redisPass, + this.clientConfig.redisUser); } else { - jedisPool = new JedisPool(poolConfig, this.clientConfig.redisHost, this.clientConfig.redisPort,2000, this.clientConfig.redisPass ); - } + jedisPool = + new JedisPool( + poolConfig, + this.clientConfig.redisHost, + this.clientConfig.redisPort, + 2000, + this.clientConfig.redisPass); + } } else { - jedisPool = new JedisPool(poolConfig, this.clientConfig.redisHost, this.clientConfig.redisPort,2000 ); + jedisPool = + new JedisPool(poolConfig, this.clientConfig.redisHost, this.clientConfig.redisPort, 2000); } } @@ -97,8 +115,9 @@ public void close() throws Exception { } } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static RedisClientConfig readConfig(File configurationFile) throws IOException { return mapper.readValue(configurationFile, RedisClientConfig.class); @@ -106,7 +125,7 @@ private static RedisClientConfig readConfig(File configurationFile) throws IOExc private static final Random random = new Random(); - private static final String getRandomString() { + private static String getRandomString() { byte[] buffer = new byte[5]; random.nextBytes(buffer); return BaseEncoding.base64Url().omitPadding().encode(buffer); diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java index 0951a9018..ead1047ac 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkProducer.java @@ -14,13 +14,14 @@ package io.openmessaging.benchmark.driver.redis; import static java.nio.charset.StandardCharsets.UTF_8; + +import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.HashMap; import java.util.Map; - import java.util.Optional; import java.util.concurrent.CompletableFuture; -import io.openmessaging.benchmark.driver.BenchmarkProducer; -import redis.clients.jedis.*; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; import redis.clients.jedis.params.XAddParams; public class RedisBenchmarkProducer implements BenchmarkProducer { @@ -44,12 +45,12 @@ public CompletableFuture sendAsync(final Optional key, final byte[ } CompletableFuture future = new CompletableFuture<>(); - try (Jedis jedis = this.pool.getResource()) { - jedis.xadd(this.rmqTopic.getBytes(UTF_8),map1, this.xaddParams); - future.complete(null); - } catch (Exception e) { - future.completeExceptionally(e); - } + try (Jedis jedis = this.pool.getResource()) { + jedis.xadd(this.rmqTopic.getBytes(UTF_8), map1, this.xaddParams); + future.complete(null); + } catch (Exception e) { + future.completeExceptionally(e); + } return future; } diff --git a/driver-rocketmq/pom.xml b/driver-rocketmq/pom.xml index 6fe1abbb6..a251ada24 100644 --- a/driver-rocketmq/pom.xml +++ b/driver-rocketmq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - messaging-benchmark io.openmessaging.benchmark + messaging-benchmark 0.0.1-SNAPSHOT - 4.0.0 driver-rocketmq @@ -35,6 +35,10 @@ driver-api ${project.version} + + com.google.guava + guava + org.apache.rocketmq rocketmq-acl @@ -50,10 +54,6 @@ rocketmq-tools ${rocketmq.version} - - com.google.guava - guava - diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java index d09475f9d..97a7e4912 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkConsumer.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.rocketmq; + import io.openmessaging.benchmark.driver.BenchmarkConsumer; import org.apache.rocketmq.client.consumer.MQPushConsumer; diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java index 8ac7c1f72..d7d547098 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver.rocketmq; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; @@ -53,10 +54,14 @@ public class RocketMQBenchmarkDriver implements BenchmarkDriver { private RPCHook rpcHook; @Override - public void initialize(final File configurationFile, final StatsLogger statsLogger) throws IOException { + public void initialize(final File configurationFile, final StatsLogger statsLogger) + throws IOException { this.rmqClientConfig = readConfig(configurationFile); if (isAclEnabled()) { - rpcHook = new AclClientRPCHook(new SessionCredentials(this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey)); + rpcHook = + new AclClientRPCHook( + new SessionCredentials( + this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey)); this.rmqAdmin = new DefaultMQAdminExt(rpcHook); } else { this.rmqAdmin = new DefaultMQAdminExt(); @@ -68,7 +73,6 @@ public void initialize(final File configurationFile, final StatsLogger statsLogg } catch (MQClientException e) { log.error("Start the RocketMQ admin tool failed."); } - } @Override @@ -78,26 +82,33 @@ public String getTopicNamePrefix() { @Override public CompletableFuture createTopic(final String topic, final int partitions) { - return CompletableFuture.runAsync(() -> { - TopicConfig topicConfig = new TopicConfig(); - topicConfig.setOrder(false); - topicConfig.setPerm(6); - topicConfig.setReadQueueNums(partitions); - topicConfig.setWriteQueueNums(partitions); - topicConfig.setTopicName(topic); - - try { - Set brokerList = CommandUtil.fetchMasterAddrByClusterName(this.rmqAdmin, this.rmqClientConfig.clusterName); - topicConfig.setReadQueueNums(Math.max(1, partitions / brokerList.size())); - topicConfig.setWriteQueueNums(Math.max(1, partitions / brokerList.size())); - - for (String brokerAddr : brokerList) { - this.rmqAdmin.createAndUpdateTopicConfig(brokerAddr, topicConfig); - } - } catch (Exception e) { - throw new RuntimeException(String.format("Failed to create topic [%s] to cluster [%s]", topic, this.rmqClientConfig.clusterName), e); - } - }); + return CompletableFuture.runAsync( + () -> { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setOrder(false); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(partitions); + topicConfig.setWriteQueueNums(partitions); + topicConfig.setTopicName(topic); + + try { + Set brokerList = + CommandUtil.fetchMasterAddrByClusterName( + this.rmqAdmin, this.rmqClientConfig.clusterName); + topicConfig.setReadQueueNums(Math.max(1, partitions / brokerList.size())); + topicConfig.setWriteQueueNums(Math.max(1, partitions / brokerList.size())); + + for (String brokerAddr : brokerList) { + this.rmqAdmin.createAndUpdateTopicConfig(brokerAddr, topicConfig); + } + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to create topic [%s] to cluster [%s]", + topic, this.rmqClientConfig.clusterName), + e); + } + }); } @Override @@ -130,11 +141,12 @@ public CompletableFuture createProducer(final String topic) { } @Override - public CompletableFuture createConsumer(final String topic, final String subscriptionName, - final ConsumerCallback consumerCallback) { + public CompletableFuture createConsumer( + final String topic, final String subscriptionName, final ConsumerCallback consumerCallback) { DefaultMQPushConsumer rmqConsumer; if (isAclEnabled()) { - rmqConsumer = new DefaultMQPushConsumer(subscriptionName, rpcHook, new AllocateMessageQueueAveragely()); + rmqConsumer = + new DefaultMQPushConsumer(subscriptionName, rpcHook, new AllocateMessageQueueAveragely()); } else { rmqConsumer = new DefaultMQPushConsumer(subscriptionName); } @@ -145,12 +157,14 @@ public CompletableFuture createConsumer(final String topic, f } try { rmqConsumer.subscribe(topic, "*"); - rmqConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> { - for (MessageExt message : msgs) { - consumerCallback.messageReceived(message.getBody(), message.getBornTimestamp()); - } - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - }); + rmqConsumer.registerMessageListener( + (MessageListenerConcurrently) + (msgs, context) -> { + for (MessageExt message : msgs) { + consumerCallback.messageReceived(message.getBody(), message.getBornTimestamp()); + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + }); rmqConsumer.start(); } catch (MQClientException e) { log.error("Failed to create consumer instance.", e); @@ -160,8 +174,8 @@ public CompletableFuture createConsumer(final String topic, f } public boolean isAclEnabled() { - return !(StringUtils.isAnyBlank(this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey) || - StringUtils.isAnyEmpty(this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey)); + return !(StringUtils.isAnyBlank(this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey) + || StringUtils.isAnyEmpty(this.rmqClientConfig.accessKey, this.rmqClientConfig.secretKey)); } @Override @@ -172,8 +186,9 @@ public void close() throws Exception { this.rmqAdmin.shutdown(); } - private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper mapper = + new ObjectMapper(new YAMLFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static RocketMQClientConfig readConfig(File configurationFile) throws IOException { return mapper.readValue(configurationFile, RocketMQClientConfig.class); @@ -181,7 +196,7 @@ private static RocketMQClientConfig readConfig(File configurationFile) throws IO private static final Random random = new Random(); - private static final String getRandomString() { + private static String getRandomString() { byte[] buffer = new byte[5]; random.nextBytes(buffer); return BaseEncoding.base64Url().omitPadding().encode(buffer); diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java index a60498933..bd80bf558 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkProducer.java @@ -13,17 +13,15 @@ */ package io.openmessaging.benchmark.driver.rocketmq; + +import io.openmessaging.benchmark.driver.BenchmarkProducer; import java.util.Optional; import java.util.concurrent.CompletableFuture; - -import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.MQProducer; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.message.Message; -import io.openmessaging.benchmark.driver.BenchmarkProducer; - public class RocketMQBenchmarkProducer implements BenchmarkProducer { private final MQProducer rmqProducer; private final String rmqTopic; @@ -42,17 +40,19 @@ public CompletableFuture sendAsync(final Optional key, final byte[ CompletableFuture future = new CompletableFuture<>(); try { - this.rmqProducer.send(message, new SendCallback() { - @Override - public void onSuccess(final SendResult sendResult) { - future.complete(null); - } + this.rmqProducer.send( + message, + new SendCallback() { + @Override + public void onSuccess(final SendResult sendResult) { + future.complete(null); + } - @Override - public void onException(final Throwable e) { - future.completeExceptionally(e); - } - }); + @Override + public void onException(final Throwable e) { + future.completeExceptionally(e); + } + }); } catch (Exception e) { future.completeExceptionally(e); } diff --git a/package/pom.xml b/package/pom.xml index 5af96eb4c..199d8f145 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - package - Packaging - pom + package + pom + Packaging - - - - maven-assembly-plugin - 3.3.0 - - - distro-assembly - package - - single - - - true - posix - openmessaging-benchmark-${project.version} - - src/assemble/bin.xml - src/assemble/src.xml - - - - - - - + + + ${project.groupId} + benchmark-framework + ${project.version} + + - - - ${project.groupId} - benchmark-framework - ${project.version} - - + + + + maven-assembly-plugin + 3.3.0 + + + distro-assembly + + single + + package + + true + posix + openmessaging-benchmark-${project.version} + + src/assemble/bin.xml + src/assemble/src.xml + + + + + + + diff --git a/pom.xml b/pom.xml index f36d00ab4..64981c973 100644 --- a/pom.xml +++ b/pom.xml @@ -1,3 +1,4 @@ + - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - io.openmessaging.benchmark - messaging-benchmark - 0.0.1-SNAPSHOT - Messaging Benchmark - pom + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + pom + Messaging Benchmark - - benchmark-framework - driver-api + 2017 - driver-pravega - driver-pulsar - driver-kafka - driver-rabbitmq - driver-artemis - driver-bookkeeper - driver-rocketmq - driver-nats - driver-nats-streaming - driver-nsq - driver-jms - driver-redis - package - docker - driver-kop - tool - + + OpenMessaging Project + https://openmessaging.cloud + - - OpenMessaging Project - https://openmessaging.cloud - + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + - 2017 + + benchmark-framework + driver-api - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - repo - - + driver-pravega + driver-pulsar + driver-kafka + driver-rabbitmq + driver-artemis + driver-bookkeeper + driver-rocketmq + driver-nats + driver-nats-streaming + driver-nsq + driver-jms + driver-redis + package + docker + driver-kop + tool + - - UTF-8 - UTF-8 + + UTF-8 + UTF-8 - 3.23.1 - 4.14.4 - 10.3.3 - 3.12.0 - 2.17.1 - 1.18.24 - 2.13.2 - 1.48 - 5.9.0 - 4.8.0 - 4.1.65.Final - 1.7.36 + 3.23.1 + 4.14.4 + 10.3.3 + 3.12.0 + 2.17.1 + 1.18.24 + 2.13.2 + 1.48 + 5.9.0 + 4.8.0 + 4.1.65.Final + 1.7.36 - 3.2.0 - 0.8.8 - 4.1 - 3.10.1 - 3.1.0 - 3.0.0-M7 - 4.7.2.0 - 2.25.0 - + 3.2.0 + 0.8.8 + 4.1 + 3.10.1 + 3.1.0 + 3.0.0-M7 + 4.7.2.0 + 2.25.0 + - - - - - com.diffplug.spotless - spotless-maven-plugin - ${spotless.plugin.version} - - - - false - true - scope,groupId,artifactId - groupId,artifactId - groupId,artifactId - - - - - - - - - - **/*.md - - - - - - - etc/**/*.xml - .github/workflows/**/*.yml - **/doc/**/*.puml - - - - - true - 4 - - - - - - - check - - check - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs.plugin.version} - - etc/findbugsExclude.xml - - - - - check - - - - - - - com.mycila - license-maven-plugin - ${license.plugin.version} - -

etc/APACHE-2.txt
- - LICENSE - NOTICE - payload/** - **/*.pyc - **/.pydevproject - .github/** - - - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - SCRIPT_STYLE - XML_STYLE - SCRIPT_STYLE - APOSTROPHE_STYLE - - - - - com.mycila - license-maven-plugin-git - ${license.plugin.version} - - - - - - check - - process-sources - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - ${checkstyle.plugin.version} - - true - true - true - etc/checkstyle.xml - true - - - - com.puppycrawl.tools - checkstyle - ${checkstyle.version} - - - - - - check - - validate - - - - - org.apache.maven.plugins - maven-enforcer-plugin - ${maven.enforcer.plugin.version} - - - enforce-maven-version - - enforce - - - - - [3.8.6,) - - - true - - - - - - maven-failsafe-plugin - ${maven.surefire.plugin.version} - - - - integration-test - verify - - - - - - maven-surefire-plugin - ${maven.surefire.plugin.version} - - - - org.jacoco - jacoco-maven-plugin - ${jacoco.plugin.version} - - - - prepare-agent - - - - report - - report - - verify - - - check - - check - - verify - - - - BUNDLE - - - CLASS - COVEREDRATIO - 0.00 - - - METHOD - COVEREDRATIO - 0.00 - - - LINE - COVEREDRATIO - 0.00 - - - BRANCH - COVEREDRATIO - 0.00 - - - - - - - - - - + + + + com.beust + jcommander + ${jcommander.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + ${jackson.version} + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-base + ${jackson.version} + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + ${jackson.version} + + + com.google.guava + guava + 29.0-jre + + + io.netty + netty-all + ${netty.version} + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.projectlombok + lombok + ${lombok.version} + provided + + + org.assertj + assertj-core + ${assertj.version} + test + + + org.junit.jupiter + junit-jupiter + ${junit.jupiter.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.junit.jupiter.version} + test + + + - - - - com.beust - jcommander - ${jcommander.version} - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-base - ${jackson.version} - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson.version} - - - com.google.guava - guava - 29.0-jre - - - io.netty - netty-all - ${netty.version} - - - org.assertj - assertj-core - ${assertj.version} - test - - - org.apache.commons - commons-lang3 - ${commons.lang3.version} - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - - org.projectlombok - lombok - ${lombok.version} - provided - - - org.junit.jupiter - junit-jupiter - ${junit.jupiter.version} - test - - - org.mockito - mockito-junit-jupiter - ${mockito.junit.jupiter.version} - test - - - + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.plugin.version} + + + + false + true + 4 + scope,groupId,artifactId + groupId,artifactId + groupId,artifactId + + + + + + + + true + 2 + + + true + 4 + + + + + **/*.md + + + + + + + etc/**/*.xml + .github/workflows/**/*.yml + **/doc/**/*.puml + + + + + true + 4 + + + + + + + check + + check + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs.plugin.version} + + etc/findbugsExclude.xml + + + + + check + + + + + + + com.mycila + license-maven-plugin + ${license.plugin.version} + +
etc/APACHE-2.txt
+ + LICENSE + NOTICE + payload/** + **/*.pyc + **/.pydevproject + .github/** + + + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + XML_STYLE + SCRIPT_STYLE + APOSTROPHE_STYLE + + + + + com.mycila + license-maven-plugin-git + ${license.plugin.version} + + + + + + check + + process-sources + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${checkstyle.plugin.version} + + true + true + true + etc/checkstyle.xml + true + + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + + + + check + + validate + + + + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven.enforcer.plugin.version} + + + enforce-maven-version + + enforce + + + + + [3.8.6,) + + + true + + + + + + maven-failsafe-plugin + ${maven.surefire.plugin.version} + + + + integration-test + verify + + + + + + maven-surefire-plugin + ${maven.surefire.plugin.version} + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.plugin.version} + + + + prepare-agent + + + + report + + report + + verify + + + check + + check + + verify + + + + BUNDLE + + + CLASS + COVEREDRATIO + 0.00 + + + METHOD + COVEREDRATIO + 0.00 + + + LINE + COVEREDRATIO + 0.00 + + + BRANCH + COVEREDRATIO + 0.00 + + + + + + + + + + - - - modern-java-compile - - [9,) - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 8 - 8 - 8 - ${project.build.sourceEncoding} - true - true - true - - - - - - - - jdk-8-compile - - [,8] - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 8 - 8 - ${project.build.sourceEncoding} - true - true - true - - - - - - - + + + modern-java-compile + + [9,) + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 8 + 8 + 8 + ${project.build.sourceEncoding} + true + true + true + + + + + + + + jdk-8-compile + + [,8] + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 8 + 8 + ${project.build.sourceEncoding} + true + true + true + + + + + + + diff --git a/tool/README.md b/tool/README.md index 9acbbfc92..231160428 100644 --- a/tool/README.md +++ b/tool/README.md @@ -6,7 +6,8 @@ Generates a set of `Workload` definition files from a `WorkloadSetTemplate` file ### Example -Template: +Template: + ```yaml nameFormat: "${topics}-topics-${partitionsPerTopic}-partitions-${messageSize}b-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}" topics: [1] @@ -22,6 +23,7 @@ testDurationMinutes: 15 ``` Usage: + ``` mkdir my-workloads io.openmessaging.benchmark.tool.workload.WorkloadGenerationTool \ @@ -30,6 +32,7 @@ mkdir my-workloads ``` Output: + ``` Starting benchmark with config: templateFile: "template.yaml" outputFolder: "my-workloads" @@ -43,6 +46,7 @@ Generated 49 workloads. ``` Example generated workload: + ```yaml name: "1-topics-1-partitions-10kb-64p-2c-50k" topics: 1 @@ -60,4 +64,5 @@ producerRate: 50000 consumerBacklogSizeGB: 0 testDurationMinutes: 5 warmupDurationMinutes: 1 -``` \ No newline at end of file +``` + diff --git a/tool/pom.xml b/tool/pom.xml index 73d344f36..97af84940 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -14,62 +14,61 @@ limitations under the License. --> - - - messaging-benchmark - io.openmessaging.benchmark - 0.0.1-SNAPSHOT - - 4.0.0 + 4.0.0 + + io.openmessaging.benchmark + messaging-benchmark + 0.0.1-SNAPSHOT + - tool + tool - - - io.openmessaging.benchmark - benchmark-framework - ${project.version} - - - org.projectlombok - lombok - - - com.beust - jcommander - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.commons - commons-text - 1.9 - - - org.apache.commons - commons-io - 1.3.2 - - - org.assertj - assertj-core - - - org.junit.jupiter - junit-jupiter - 5.9.0 - test - - - org.mockito - mockito-junit-jupiter - 4.7.0 - test - - + + + com.beust + jcommander + + + io.openmessaging.benchmark + benchmark-framework + ${project.version} + + + org.apache.commons + commons-io + 1.3.2 + + + org.apache.commons + commons-text + 1.9 + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.assertj + assertj-core + + + org.projectlombok + lombok + + + org.junit.jupiter + junit-jupiter + 5.9.0 + test + + + org.mockito + mockito-junit-jupiter + 4.7.0 + test + + diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerationTool.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerationTool.java index 0c11323b4..91bfb7d5d 100644 --- a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerationTool.java +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerationTool.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.tool.workload; + import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; @@ -26,14 +27,13 @@ import java.util.List; import lombok.extern.slf4j.Slf4j; -/** - * Generates a set of {@link Workload} definition files from a {@link WorkloadSetTemplate} file. - */ +/** Generates a set of {@link Workload} definition files from a {@link WorkloadSetTemplate} file. */ @Slf4j public class WorkloadGenerationTool { private static final ObjectMapper mapper = - new ObjectMapper(new YAMLFactory().configure(YAMLGenerator.Feature.WRITE_DOC_START_MARKER, false)) + new ObjectMapper( + new YAMLFactory().configure(YAMLGenerator.Feature.WRITE_DOC_START_MARKER, false)) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); static { @@ -61,7 +61,8 @@ public static void main(String[] args) throws IOException { // Dump configuration variables log.info("Starting benchmark with config: {}", mapper.writeValueAsString(arguments)); - WorkloadSetTemplate template = mapper.readValue(arguments.templateFile, WorkloadSetTemplate.class); + WorkloadSetTemplate template = + mapper.readValue(arguments.templateFile, WorkloadSetTemplate.class); List workloads = new WorkloadGenerator(template).generate(); for (Workload w : workloads) { File outputFile = null; @@ -75,14 +76,22 @@ public static void main(String[] args) throws IOException { } static class Arguments { - @Parameter(names = {"-t", "--template-file"}, - description = "Path to a YAML file containing the workload template", required = true) + @Parameter( + names = {"-t", "--template-file"}, + description = "Path to a YAML file containing the workload template", + required = true) public File templateFile; - @Parameter(names = {"-o", "--output-folder"}, description = "Output", required = true) + @Parameter( + names = {"-o", "--output-folder"}, + description = "Output", + required = true) public File outputFolder; - @Parameter(names = {"-h", "--help"}, description = "Help message", help = true) + @Parameter( + names = {"-h", "--help"}, + description = "Help message", + help = true) boolean help; } } diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerator.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerator.java index 7e6d77b8c..d2bfc7956 100644 --- a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerator.java +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadGenerator.java @@ -14,6 +14,7 @@ package io.openmessaging.benchmark.tool.workload; import static java.util.Collections.unmodifiableList; + import io.openmessaging.benchmark.Workload; import java.io.IOException; import java.util.ArrayList; diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormat.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormat.java index 6a97b0c27..501d6fc00 100644 --- a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormat.java +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormat.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.tool.workload; + import io.openmessaging.benchmark.Workload; import java.util.HashMap; import java.util.Map; @@ -20,9 +21,10 @@ import org.apache.commons.lang.text.StrSubstitutor; /** - * Generates {@link Workload} names based on a template. Substitutes template place-holders of the form - * {@code ${variableName}}, where {@code variableName} is the name of a public member in {@link Workload}. Note that the - * set of variables is statically assigned. Numeric values will typically be in a form that includes an SI suffix. + * Generates {@link Workload} names based on a template. Substitutes template place-holders of the + * form {@code ${variableName}}, where {@code variableName} is the name of a public member in {@link + * Workload}. Note that the set of variables is statically assigned. Numeric values will typically + * be in a form that includes an SI suffix. */ @RequiredArgsConstructor class WorkloadNameFormat { @@ -42,8 +44,11 @@ String from(Workload workload) { params.put("subscriptionsPerTopic", countToDisplaySize(workload.subscriptionsPerTopic)); params.put("producersPerTopic", countToDisplaySize(workload.producersPerTopic)); params.put("consumerPerSubscription", countToDisplaySize(workload.consumerPerSubscription)); - params.put("producerRate", - (workload.producerRate >= MAX_PRODUCER_RATE) ? "max-rate" : countToDisplaySize(workload.producerRate)); + params.put( + "producerRate", + (workload.producerRate >= MAX_PRODUCER_RATE) + ? "max-rate" + : countToDisplaySize(workload.producerRate)); params.put("keyDistributor", workload.keyDistributor); params.put("payloadFile", workload.payloadFile); params.put("useRandomizedPayloads", workload.useRandomizedPayloads); diff --git a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java index 3ce018847..538a28ef4 100644 --- a/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java +++ b/tool/src/main/java/io/openmessaging/benchmark/tool/workload/WorkloadSetTemplate.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.tool.workload; + import io.openmessaging.benchmark.utils.distributor.KeyDistributorType; import java.util.Collections; import java.util.List; @@ -20,27 +21,28 @@ import lombok.NoArgsConstructor; /** - * A template that defines a set of workload definitions. This is much like the - * {@link io.openmessaging.benchmark.Workload} entity, except that for many values that typically change in a benchmark, - * one can specify a sequence of values. + * A template that defines a set of workload definitions. This is much like the {@link + * io.openmessaging.benchmark.Workload} entity, except that for many values that typically change in + * a benchmark, one can specify a sequence of values. */ @Data @NoArgsConstructor public class WorkloadSetTemplate { public static final String DEFAULT_NAME_TEMPLATE = "${topics}-topics-${partitionsPerTopic}-partitions-${messageSize}b" - + "-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}"; + + "-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}"; public String nameFormat = DEFAULT_NAME_TEMPLATE; - /** Number of topics to create in the test */ + /** Number of topics to create in the test. */ public List topics = Collections.emptyList(); - /** Number of partitions each topic will contain */ - public List partitionsPerTopic= Collections.emptyList(); - public List messageSize= Collections.emptyList(); - public List subscriptionsPerTopic= Collections.emptyList(); - public List producersPerTopic= Collections.emptyList(); - public List consumerPerSubscription= Collections.emptyList(); - public List producerRate= Collections.emptyList(); + /** Number of partitions each topic will contain. */ + public List partitionsPerTopic = Collections.emptyList(); + + public List messageSize = Collections.emptyList(); + public List subscriptionsPerTopic = Collections.emptyList(); + public List producersPerTopic = Collections.emptyList(); + public List consumerPerSubscription = Collections.emptyList(); + public List producerRate = Collections.emptyList(); public KeyDistributorType keyDistributor = KeyDistributorType.NO_KEY; public String payloadFile = null; diff --git a/tool/src/test/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormatTest.java b/tool/src/test/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormatTest.java index 6c0d7ae40..63ff456fc 100644 --- a/tool/src/test/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormatTest.java +++ b/tool/src/test/java/io/openmessaging/benchmark/tool/workload/WorkloadNameFormatTest.java @@ -20,8 +20,9 @@ class WorkloadNameFormatTest { - public String nameFormat = "${topics}-topics-${partitionsPerTopic}-partitions-${messageSize}b" - + "-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}"; + public String nameFormat = + "${topics}-topics-${partitionsPerTopic}-partitions-${messageSize}b" + + "-${producersPerTopic}p-${consumerPerSubscription}c-${producerRate}"; @Test void nameOverride() { @@ -43,4 +44,4 @@ void from() { String name = new WorkloadNameFormat(nameFormat).from(workload); assertThat(name).isEqualTo("1k-topics-2k-partitions-617kb-45p-541c-1m"); } -} \ No newline at end of file +} From 3d88d7ba36fd906ff3ec725734d6a221417a9d1a Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Fri, 30 Sep 2022 16:43:52 +0100 Subject: [PATCH 066/103] Keep running for testDurationMinutes after a backlog drain (#328) ## Motivation When running a backlog drain workload, if the time to build and drain the backlog exceeds the `testDurationMinutes` then the benchmark will immediately terminate when the drain is complete. This can truncate the collection of useful metrics. ## Changes * Keep running the benchmark for `testDurationMinutes` after the backlog has been drained. --- .../openmessaging/benchmark/WorkloadGenerator.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 6905fdf22..721565ad0 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark; +import static java.util.concurrent.TimeUnit.MINUTES; import io.netty.util.concurrent.DefaultThreadFactory; import io.openmessaging.benchmark.utils.PaddingDecimalFormat; @@ -129,7 +130,7 @@ public TestResult run() throws Exception { executor.execute( () -> { try { - buildAndDrainBacklog(topics); + buildAndDrainBacklog(workload.testDurationMinutes); } catch (IOException e) { e.printStackTrace(); } @@ -358,7 +359,7 @@ private void createProducers(List topics) throws IOException { log.info("Created {} producers in {} ms", fullListOfTopics.size(), timer.elapsedMillis()); } - private void buildAndDrainBacklog(List topics) throws IOException { + private void buildAndDrainBacklog(int testDurationMinutes) throws IOException { Timer timer = new Timer(); log.info("Stopping all consumers to build backlog"); worker.pauseConsumers(); @@ -398,6 +399,13 @@ private void buildAndDrainBacklog(List topics) throws IOException { workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived; if (currentBacklog <= minBacklog) { log.info("--- Completed backlog draining in {} s ---", timer.elapsedSeconds()); + + try { + Thread.sleep(MINUTES.toMillis(testDurationMinutes)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + needToWaitForBacklogDraining = false; return; } From 8f7d5d65ef63d87140b5908945df4b2bfdd4645a Mon Sep 17 00:00:00 2001 From: Elliot West Date: Fri, 30 Sep 2022 16:48:33 +0100 Subject: [PATCH 067/103] [Refactor] Break up ensemble to enable testing (#327) * Factor out HTTP client to enable testing * Header, reduce * Move aggregation methods for easier testing * Local worker ID * Static imports * Method reference * ignore exception * Ignore * Update HDR version * Share paths * More constants * Rename * Use this * Parallel * Use constant * Add test * Rename * Short form * Histogram SerDe * Spotbugs exclusions * Tests * Test Histogram SerDe * Tidy usage * Fix error * Method reference * Test * Factor out object mappers * Spotbugs exclude * Spotbugs exclude * comment * typo * Reformat code * Decompose LocalWorker * Fix bug --- benchmark-framework/pom.xml | 14 +- .../io/openmessaging/benchmark/Benchmark.java | 6 +- .../openmessaging/benchmark/utils/Timer.java | 2 +- .../benchmark/utils/UniformRateLimiter.java | 6 +- .../worker/DistributedWorkersEnsemble.java | 441 ++++++------------ .../benchmark/worker/HttpWorkerClient.java | 231 +++++++++ .../benchmark/worker/LocalWorker.java | 196 +------- .../benchmark/worker/MessageProducer.java | 63 +++ .../benchmark/worker/Worker.java | 2 + .../benchmark/worker/WorkerHandler.java | 109 ++--- .../benchmark/worker/WorkerStats.java | 186 ++++++++ .../worker/commands/CountersStats.java | 12 + .../worker/commands/CumulativeLatencies.java | 25 +- .../worker/commands/PeriodStats.java | 41 +- .../commands/ProducerWorkAssignment.java | 8 + .../worker/jackson/HistogramDeserializer.java | 63 +++ .../worker/jackson/HistogramSerializer.java | 45 ++ .../worker/jackson/ObjectMappers.java | 46 ++ .../DistributedWorkersEnsembleTest.java | 56 +++ .../worker/commands/CountersStatsTest.java | 60 +++ .../commands/CumulativeLatenciesTest.java | 35 ++ .../worker/commands/PeriodStatsTest.java | 95 ++++ .../worker/jackson/HistogramSerDeTest.java | 44 ++ etc/findbugsExclude.xml | 20 + 24 files changed, 1237 insertions(+), 569 deletions(-) create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/HttpWorkerClient.java create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/MessageProducer.java create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerStats.java create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/ObjectMappers.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsembleTest.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CountersStatsTest.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CumulativeLatenciesTest.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/PeriodStatsTest.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index 1c6b2b8bf..ea9949af9 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -119,9 +119,8 @@ log4j-slf4j-impl - org.asynchttpclient - async-http-client - 2.12.3 + org.assertj + assertj-core org.asynchttpclient @@ -142,22 +141,15 @@ org.hdrhistogram HdrHistogram - 2.1.10 - - - org.assertj - assertj-core - test + 2.1.12 org.junit.jupiter junit-jupiter - test org.mockito mockito-junit-jupiter - test diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index 25c5c5a9f..d18cf90d7 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark; +import static java.util.stream.Collectors.toList; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -22,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.openmessaging.benchmark.worker.DistributedWorkersEnsemble; +import io.openmessaging.benchmark.worker.HttpWorkerClient; import io.openmessaging.benchmark.worker.LocalWorker; import io.openmessaging.benchmark.worker.Worker; import java.io.File; @@ -138,7 +140,9 @@ public static void main(String[] args) throws Exception { Worker worker; if (arguments.workers != null && !arguments.workers.isEmpty()) { - worker = new DistributedWorkersEnsemble(arguments.workers, arguments.extraConsumers); + List workers = + arguments.workers.stream().map(w -> new HttpWorkerClient(w)).collect(toList()); + worker = new DistributedWorkersEnsemble(workers, arguments.extraConsumers); } else { // Use local worker implementation worker = new LocalWorker(); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java index 11a49de44..bb3ac28c0 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Timer.java @@ -27,7 +27,7 @@ public class Timer { } public Timer() { - this(() -> System.nanoTime()); + this(System::nanoTime); } public double elapsedMillis() { diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java index 75ae74797..1fc69b15c 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/UniformRateLimiter.java @@ -13,8 +13,8 @@ */ package io.openmessaging.benchmark.utils; +import static java.util.concurrent.TimeUnit.SECONDS; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.locks.LockSupport; import java.util.function.Supplier; @@ -29,7 +29,7 @@ public final class UniformRateLimiter { AtomicLongFieldUpdater.newUpdater(UniformRateLimiter.class, "virtualTime"); private static final AtomicLongFieldUpdater START_UPDATER = AtomicLongFieldUpdater.newUpdater(UniformRateLimiter.class, "start"); - private static final double ONE_SEC_IN_NS = TimeUnit.SECONDS.toNanos(1); + private static final double ONE_SEC_IN_NS = SECONDS.toNanos(1); private volatile long start = Long.MIN_VALUE; private volatile long virtualTime; private final double opsPerSec; @@ -49,7 +49,7 @@ public final class UniformRateLimiter { } public UniformRateLimiter(final double opsPerSec) { - this(opsPerSec, () -> System.nanoTime()); + this(opsPerSec, System::nanoTime); } public double getOpsPerSec() { diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index a27628851..c41a0da51 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -14,17 +14,12 @@ package io.openmessaging.benchmark.worker; import static java.util.Collections.unmodifiableList; -import static java.util.stream.Collectors.toList; -import static org.asynchttpclient.Dsl.asyncHttpClient; +import static java.util.stream.Collectors.joining; import com.beust.jcommander.internal.Maps; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import io.netty.buffer.ByteBufUtil; -import io.netty.buffer.Unpooled; import io.openmessaging.benchmark.utils.ListPartition; import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; import io.openmessaging.benchmark.worker.commands.CountersStats; @@ -35,74 +30,73 @@ import io.openmessaging.benchmark.worker.commands.TopicsInfo; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; -import java.util.zip.DataFormatException; -import org.HdrHistogram.Histogram; -import org.asynchttpclient.AsyncHttpClient; -import org.asynchttpclient.DefaultAsyncHttpClientConfig; -import org.asynchttpclient.Dsl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DistributedWorkersEnsemble implements Worker { private final Thread shutdownHook = new Thread(this::stopAll); - private final List workers; - private final List producerWorkers; - private final List consumerWorkers; - - private final AsyncHttpClient httpClient; + private final List workers; + private final List producerWorkers; + private final List consumerWorkers; + private final Worker leader; private int numberOfUsedProducerWorkers; - public DistributedWorkersEnsemble(List workers, boolean extraConsumerWorkers) { + public DistributedWorkersEnsemble(List workers, boolean extraConsumerWorkers) { Preconditions.checkArgument(workers.size() > 1); - DefaultAsyncHttpClientConfig.Builder clientBuilder = - Dsl.config().setReadTimeout(600000).setRequestTimeout(600000); - httpClient = asyncHttpClient(clientBuilder); this.workers = unmodifiableList(workers); - - // For driver-jms extra consumers are required. - // If there is an odd number of workers then allocate the extra to consumption. - int numberOfProducerWorkers = - extraConsumerWorkers ? (workers.size() + 2) / 3 : workers.size() / 2; - List> partitions = + leader = workers.get(0); + int numberOfProducerWorkers = getNumberOfProducerWorkers(workers, extraConsumerWorkers); + List> partitions = Lists.partition(Lists.reverse(workers), workers.size() - numberOfProducerWorkers); this.producerWorkers = partitions.get(1); this.consumerWorkers = partitions.get(0); - log.info("Workers list - producers: {}", producerWorkers); - log.info("Workers list - consumers: {}", consumerWorkers); + log.info( + "Workers list - producers: [{}]", + producerWorkers.stream().map(Worker::id).collect(joining(","))); + log.info( + "Workers list - consumers: {}", + consumerWorkers.stream().map(Worker::id).collect(joining(","))); Runtime.getRuntime().addShutdownHook(shutdownHook); } + /* + * For driver-jms extra consumers are required. If there is an odd number of workers then allocate the extra + * to consumption. + */ + @VisibleForTesting + static int getNumberOfProducerWorkers(List workers, boolean extraConsumerWorkers) { + return extraConsumerWorkers ? (workers.size() + 2) / 3 : workers.size() / 2; + } + @Override public void initializeDriver(File configurationFile) throws IOException { - byte[] confFileContent = Files.readAllBytes(Paths.get(configurationFile.toString())); - sendPost(workers, "/initialize-driver", confFileContent); + workers.parallelStream() + .forEach( + w -> { + try { + w.initializeDriver(configurationFile); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } @Override @SuppressWarnings("unchecked") public List createTopics(TopicsInfo topicsInfo) throws IOException { - // Create all topics from a single worker node - return (List) - post(workers.get(0), "/create-topics", writer.writeValueAsBytes(topicsInfo), List.class) - .join(); + return leader.createTopics(topicsInfo); } @Override public void createProducers(List topics) { List> topicsPerProducer = ListPartition.partitionList(topics, producerWorkers.size()); - Map> topicsPerProducerMap = Maps.newHashMap(); + Map> topicsPerProducerMap = Maps.newHashMap(); int i = 0; for (List assignedTopics : topicsPerProducer) { topicsPerProducerMap.put(producerWorkers.get(i++), assignedTopics); @@ -113,62 +107,96 @@ public void createProducers(List topics) { (int) topicsPerProducerMap.values().stream().filter(t -> !t.isEmpty()).count(); log.debug( "Producing worker count: {} of {}", numberOfUsedProducerWorkers, producerWorkers.size()); - - CompletableFuture[] futures = - topicsPerProducerMap.keySet().stream() - .map( - producer -> { - try { - return sendPost( - producer, - "/create-producers", - writer.writeValueAsBytes(topicsPerProducerMap.get(producer))); - } catch (Exception e) { - CompletableFuture future = new CompletableFuture<>(); - future.completeExceptionally(e); - return future; - } - }) - .toArray(this::newArray); - - CompletableFuture.allOf(futures).join(); + topicsPerProducerMap.entrySet().parallelStream() + .forEach( + e -> { + try { + e.getKey().createProducers(e.getValue()); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); } @Override public void startLoad(ProducerWorkAssignment producerWorkAssignment) throws IOException { // Reduce the publish rate across all the brokers - producerWorkAssignment.publishRate /= numberOfUsedProducerWorkers; - log.debug( - "Setting worker assigned publish rate to {} msgs/sec", producerWorkAssignment.publishRate); - sendPost(producerWorkers, "/start-load", writer.writeValueAsBytes(producerWorkAssignment)); + double newRate = producerWorkAssignment.publishRate / numberOfUsedProducerWorkers; + log.debug("Setting worker assigned publish rate to {} msgs/sec", newRate); + // Reduce the publish rate across all the brokers + producerWorkers.parallelStream() + .forEach( + w -> { + try { + w.startLoad(producerWorkAssignment.withPublishRate(newRate)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } @Override public void probeProducers() throws IOException { - sendPost(producerWorkers, "/probe-producers", new byte[0]); + producerWorkers.parallelStream() + .forEach( + w -> { + try { + w.probeProducers(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } @Override public void adjustPublishRate(double publishRate) throws IOException { - // Reduce the publish rate across all the brokers - publishRate /= numberOfUsedProducerWorkers; - log.debug("Adjusting worker publish rate to {} msgs/sec", publishRate); - sendPost(producerWorkers, "/adjust-publish-rate", writer.writeValueAsBytes(publishRate)); + double newRate = publishRate / numberOfUsedProducerWorkers; + log.debug("Adjusting producer publish rate to {} msgs/sec", newRate); + producerWorkers.parallelStream() + .forEach( + w -> { + try { + w.adjustPublishRate(newRate); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } @Override public void stopAll() { - sendPost(workers, "/stop-all", new byte[0]); + workers.parallelStream().forEach(Worker::stopAll); + } + + @Override + public String id() { + return "Ensemble[" + workers.stream().map(Worker::id).collect(joining(",")) + "]"; } @Override public void pauseConsumers() throws IOException { - sendPost(consumerWorkers, "/pause-consumers", new byte[0]); + consumerWorkers.parallelStream() + .forEach( + w -> { + try { + w.pauseConsumers(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } @Override public void resumeConsumers() throws IOException { - sendPost(consumerWorkers, "/resume-consumers", new byte[0]); + consumerWorkers.parallelStream() + .forEach( + w -> { + try { + w.resumeConsumers(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } @Override @@ -176,228 +204,73 @@ public void createConsumers(ConsumerAssignment overallConsumerAssignment) { List> subscriptionsPerConsumer = ListPartition.partitionList( overallConsumerAssignment.topicsSubscriptions, consumerWorkers.size()); - Map topicsPerWorkerMap = Maps.newHashMap(); + Map topicsPerWorkerMap = Maps.newHashMap(); int i = 0; for (List tsl : subscriptionsPerConsumer) { - ConsumerAssignment individualAssignement = new ConsumerAssignment(); - individualAssignement.topicsSubscriptions = tsl; - topicsPerWorkerMap.put(consumerWorkers.get(i++), individualAssignement); + ConsumerAssignment individualAssignment = new ConsumerAssignment(); + individualAssignment.topicsSubscriptions = tsl; + topicsPerWorkerMap.put(consumerWorkers.get(i++), individualAssignment); } - - CompletableFuture[] futures = - topicsPerWorkerMap.keySet().stream() - .map( - consumer -> { - try { - return sendPost( - consumer, - "/create-consumers", - writer.writeValueAsBytes(topicsPerWorkerMap.get(consumer))); - } catch (Exception e) { - CompletableFuture future = new CompletableFuture<>(); - future.completeExceptionally(e); - return future; - } - }) - .toArray(this::newArray); - - CompletableFuture.allOf(futures).join(); + topicsPerWorkerMap.entrySet().parallelStream() + .forEach( + e -> { + try { + e.getKey().createConsumers(e.getValue()); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); } @Override public PeriodStats getPeriodStats() { - List individualStats = get(workers, "/period-stats", PeriodStats.class); - PeriodStats stats = new PeriodStats(); - individualStats.forEach( - is -> { - stats.messagesSent += is.messagesSent; - stats.messageSendErrors += is.messageSendErrors; - stats.bytesSent += is.bytesSent; - stats.messagesReceived += is.messagesReceived; - stats.bytesReceived += is.bytesReceived; - stats.totalMessagesSent += is.totalMessagesSent; - stats.totalMessageSendErrors += is.totalMessageSendErrors; - stats.totalMessagesReceived += is.totalMessagesReceived; - - try { - stats.publishLatency.add( - Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30))); - - stats.publishDelayLatency.add( - Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.publishDelayLatencyBytes), TimeUnit.SECONDS.toMicros(30))); - - stats.endToEndLatency.add( - Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); - } catch (ArrayIndexOutOfBoundsException | DataFormatException e) { - throw new RuntimeException(e); - } - }); - - return stats; + return workers.parallelStream() + .map( + w -> { + try { + return w.getPeriodStats(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .reduce(new PeriodStats(), PeriodStats::plus); } @Override public CumulativeLatencies getCumulativeLatencies() { - List individualStats = - get(workers, "/cumulative-latencies", CumulativeLatencies.class); - - CumulativeLatencies stats = new CumulativeLatencies(); - individualStats.forEach( - is -> { - try { - stats.publishLatency.add( - Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30))); - } catch (Exception e) { - log.error( - "Failed to decode publish latency: {}", - ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.publishLatencyBytes))); - throw new RuntimeException(e); - } - - try { - stats.publishDelayLatency.add( - Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.publishDelayLatencyBytes), TimeUnit.SECONDS.toMicros(30))); - } catch (Exception e) { - log.error( - "Failed to decode publish delay latency: {}", - ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.publishDelayLatencyBytes))); - throw new RuntimeException(e); - } - - try { - stats.endToEndLatency.add( - Histogram.decodeFromCompressedByteBuffer( - ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); - } catch (Exception e) { - log.error( - "Failed to decode end-to-end latency: {}", - ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(is.endToEndLatencyBytes))); - throw new RuntimeException(e); - } - }); - - return stats; - } - - @Override - public CountersStats getCountersStats() throws IOException { - List individualStats = get(workers, "/counters-stats", CountersStats.class); - - CountersStats stats = new CountersStats(); - individualStats.forEach( - is -> { - stats.messagesSent += is.messagesSent; - stats.messagesReceived += is.messagesReceived; - stats.messageSendErrors += is.messageSendErrors; - }); - - return stats; - } - - @Override - public void resetStats() throws IOException { - sendPost(workers, "/reset-stats", new byte[0]); - } - - /** - * Send a request to multiple hosts and wait for all responses. - * - * @param hosts - * @param path - * @param body - */ - private void sendPost(List hosts, String path, byte[] body) { - CompletableFuture[] futures = - hosts.stream().map(w -> sendPost(w, path, body)).toArray(this::newArray); - CompletableFuture.allOf(futures).join(); - } - - private CompletableFuture sendPost(String host, String path, byte[] body) { - return httpClient - .preparePost(host + path) - .setBody(body) - .execute() - .toCompletableFuture() - .thenApply( - x -> { - if (x.getStatusCode() != 200) { - log.error( - "Failed to do HTTP post request to {}{} -- code: {}", - host, - path, - x.getStatusCode()); + return workers.parallelStream() + .map( + w -> { + try { + return w.getCumulativeLatencies(); + } catch (IOException e) { + throw new RuntimeException(e); } - Preconditions.checkArgument(x.getStatusCode() == 200); - return (Void) null; - }); - } - - private List get(List hosts, String path, Class clazz) { - CompletableFuture[] futures = - hosts.stream().map(w -> get(w, path, clazz)).toArray(this::newArray); - - CompletableFuture> resultFuture = new CompletableFuture<>(); - CompletableFuture.allOf(futures) - .thenRun( - () -> { - resultFuture.complete( - Stream.of(futures).map(CompletableFuture::join).collect(toList())); }) - .exceptionally( - ex -> { - resultFuture.completeExceptionally(ex); - return null; - }); - - return resultFuture.join(); + .reduce(new CumulativeLatencies(), CumulativeLatencies::plus); } - private CompletableFuture get(String host, String path, Class clazz) { - return httpClient - .prepareGet(host + path) - .execute() - .toCompletableFuture() - .thenApply( - response -> { + @Override + public CountersStats getCountersStats() throws IOException { + return workers.parallelStream() + .map( + w -> { try { - if (response.getStatusCode() != 200) { - log.error( - "Failed to do HTTP get request to {}{} -- code: {}", - host, - path, - response.getStatusCode()); - } - Preconditions.checkArgument(response.getStatusCode() == 200); - return mapper.readValue(response.getResponseBody(), clazz); + return w.getCountersStats(); } catch (IOException e) { throw new RuntimeException(e); } - }); + }) + .reduce(new CountersStats(), CountersStats::plus); } - private CompletableFuture post(String host, String path, byte[] body, Class clazz) { - return httpClient - .preparePost(host + path) - .setBody(body) - .execute() - .toCompletableFuture() - .thenApply( - response -> { + @Override + public void resetStats() throws IOException { + workers.parallelStream() + .forEach( + w -> { try { - if (response.getStatusCode() != 200) { - log.error( - "Failed to do HTTP post request to {}{} -- code: {}", - host, - path, - response.getStatusCode()); - } - Preconditions.checkArgument(response.getStatusCode() == 200); - return mapper.readValue(response.getResponseBody(), clazz); + w.resetStats(); } catch (IOException e) { throw new RuntimeException(e); } @@ -407,21 +280,13 @@ private CompletableFuture post(String host, String path, byte[] body, Cla @Override public void close() throws Exception { Runtime.getRuntime().removeShutdownHook(shutdownHook); - httpClient.close(); - } - - @SuppressWarnings("unchecked") - private CompletableFuture[] newArray(int size) { - return new CompletableFuture[size]; - } - - private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter(); - - private static final ObjectMapper mapper = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - static { - mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE); + for (Worker w : workers) { + try { + w.close(); + } catch (Exception ignored) { + log.trace("Ignored error while closing worker {}", w, ignored); + } + } } private static final Logger log = LoggerFactory.getLogger(DistributedWorkersEnsemble.class); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/HttpWorkerClient.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/HttpWorkerClient.java new file mode 100644 index 000000000..513b04178 --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/HttpWorkerClient.java @@ -0,0 +1,231 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker; + +import static io.openmessaging.benchmark.worker.WorkerHandler.ADJUST_PUBLISH_RATE; +import static io.openmessaging.benchmark.worker.WorkerHandler.COUNTERS_STATS; +import static io.openmessaging.benchmark.worker.WorkerHandler.CREATE_CONSUMERS; +import static io.openmessaging.benchmark.worker.WorkerHandler.CREATE_PRODUCERS; +import static io.openmessaging.benchmark.worker.WorkerHandler.CREATE_TOPICS; +import static io.openmessaging.benchmark.worker.WorkerHandler.CUMULATIVE_LATENCIES; +import static io.openmessaging.benchmark.worker.WorkerHandler.INITIALIZE_DRIVER; +import static io.openmessaging.benchmark.worker.WorkerHandler.PAUSE_CONSUMERS; +import static io.openmessaging.benchmark.worker.WorkerHandler.PERIOD_STATS; +import static io.openmessaging.benchmark.worker.WorkerHandler.PROBE_PRODUCERS; +import static io.openmessaging.benchmark.worker.WorkerHandler.RESET_STATS; +import static io.openmessaging.benchmark.worker.WorkerHandler.RESUME_CONSUMERS; +import static io.openmessaging.benchmark.worker.WorkerHandler.START_LOAD; +import static io.openmessaging.benchmark.worker.WorkerHandler.STOP_ALL; +import static org.asynchttpclient.Dsl.asyncHttpClient; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.common.base.Preconditions; +import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; +import io.openmessaging.benchmark.worker.commands.CountersStats; +import io.openmessaging.benchmark.worker.commands.CumulativeLatencies; +import io.openmessaging.benchmark.worker.commands.PeriodStats; +import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment; +import io.openmessaging.benchmark.worker.commands.TopicsInfo; +import io.openmessaging.benchmark.worker.jackson.ObjectMappers; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import org.asynchttpclient.AsyncHttpClient; +import org.asynchttpclient.Dsl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HttpWorkerClient implements Worker { + + private static final byte[] EMPTY_BODY = new byte[0]; + private static final int HTTP_OK = 200; + + private final AsyncHttpClient httpClient; + private final String host; + + public HttpWorkerClient(String host) { + this(asyncHttpClient(Dsl.config().setReadTimeout(600000).setRequestTimeout(600000)), host); + } + + HttpWorkerClient(AsyncHttpClient httpClient, String host) { + this.httpClient = httpClient; + this.host = host; + } + + @Override + public void initializeDriver(File configurationFile) throws IOException { + byte[] confFileContent = Files.readAllBytes(Paths.get(configurationFile.toString())); + sendPost(INITIALIZE_DRIVER, confFileContent); + } + + @SuppressWarnings("unchecked") + @Override + public List createTopics(TopicsInfo topicsInfo) throws IOException { + return (List) post(CREATE_TOPICS, writer.writeValueAsBytes(topicsInfo), List.class); + } + + @Override + public void createProducers(List topics) throws IOException { + sendPost(CREATE_PRODUCERS, writer.writeValueAsBytes(topics)); + } + + @Override + public void createConsumers(ConsumerAssignment consumerAssignment) throws IOException { + sendPost(CREATE_CONSUMERS, writer.writeValueAsBytes(consumerAssignment)); + } + + @Override + public void probeProducers() throws IOException { + sendPost(PROBE_PRODUCERS); + } + + @Override + public void startLoad(ProducerWorkAssignment producerWorkAssignment) throws IOException { + log.debug( + "Setting worker assigned publish rate to {} msgs/sec", producerWorkAssignment.publishRate); + sendPost(START_LOAD, writer.writeValueAsBytes(producerWorkAssignment)); + } + + @Override + public void adjustPublishRate(double publishRate) throws IOException { + log.debug("Adjusting worker publish rate to {} msgs/sec", publishRate); + sendPost(ADJUST_PUBLISH_RATE, writer.writeValueAsBytes(publishRate)); + } + + @Override + public void pauseConsumers() throws IOException { + sendPost(PAUSE_CONSUMERS); + } + + @Override + public void resumeConsumers() throws IOException { + sendPost(RESUME_CONSUMERS); + } + + @Override + public CountersStats getCountersStats() throws IOException { + return get(COUNTERS_STATS, CountersStats.class); + } + + @Override + public PeriodStats getPeriodStats() throws IOException { + return get(PERIOD_STATS, PeriodStats.class); + } + + @Override + public CumulativeLatencies getCumulativeLatencies() throws IOException { + return get(CUMULATIVE_LATENCIES, CumulativeLatencies.class); + } + + @Override + public void resetStats() throws IOException { + sendPost(RESET_STATS); + } + + @Override + public void stopAll() { + sendPost(STOP_ALL); + } + + @Override + public String id() { + return host; + } + + @Override + public void close() throws Exception { + httpClient.close(); + } + + private void sendPost(String path) { + sendPost(path, EMPTY_BODY); + } + + private void sendPost(String path, byte[] body) { + httpClient + .preparePost(host + path) + .setBody(body) + .execute() + .toCompletableFuture() + .thenApply( + response -> { + if (response.getStatusCode() != HTTP_OK) { + log.error( + "Failed to do HTTP post request to {}{} -- code: {}", + host, + path, + response.getStatusCode()); + } + Preconditions.checkArgument(response.getStatusCode() == HTTP_OK); + return (Void) null; + }) + .join(); + } + + private T get(String path, Class clazz) { + return httpClient + .prepareGet(host + path) + .execute() + .toCompletableFuture() + .thenApply( + response -> { + try { + if (response.getStatusCode() != HTTP_OK) { + log.error( + "Failed to do HTTP get request to {}{} -- code: {}", + host, + path, + response.getStatusCode()); + } + Preconditions.checkArgument(response.getStatusCode() == HTTP_OK); + return mapper.readValue(response.getResponseBody(), clazz); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .join(); + } + + private T post(String path, byte[] body, Class clazz) { + return httpClient + .preparePost(host + path) + .setBody(body) + .execute() + .toCompletableFuture() + .thenApply( + response -> { + try { + if (response.getStatusCode() != HTTP_OK) { + log.error( + "Failed to do HTTP post request to {}{} -- code: {}", + host, + path, + response.getStatusCode()); + } + Preconditions.checkArgument(response.getStatusCode() == HTTP_OK); + return mapper.readValue(response.getResponseBody(), clazz); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .join(); + } + + private static final ObjectMapper mapper = ObjectMappers.DEFAULT.mapper(); + private static final ObjectWriter writer = ObjectMappers.DEFAULT.writer(); + private static final Logger log = LoggerFactory.getLogger(HttpWorkerClient.class); +} diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index ca5d20b69..8a5c57278 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -13,7 +13,6 @@ */ package io.openmessaging.benchmark.worker; -import static io.openmessaging.benchmark.utils.UniformRateLimiter.uninterruptibleSleepNs; import static java.util.stream.Collectors.toList; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -51,12 +50,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.LongAdder; import java.util.stream.IntStream; -import org.HdrHistogram.Recorder; -import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.NullStatsLogger; -import org.apache.bookkeeper.stats.OpStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,52 +59,13 @@ public class LocalWorker implements Worker, ConsumerCallback { private BenchmarkDriver benchmarkDriver = null; - - private List producers = new ArrayList<>(); - private List consumers = new ArrayList<>(); - + private final List producers = new ArrayList<>(); + private final List consumers = new ArrayList<>(); private volatile UniformRateLimiter rateLimiter = new UniformRateLimiter(1.0); - private final ExecutorService executor = Executors.newCachedThreadPool(new DefaultThreadFactory("local-worker")); - - // stats - - private final StatsLogger statsLogger; - - private final LongAdder messagesSent = new LongAdder(); - private final LongAdder messageSendErrors = new LongAdder(); - private final LongAdder bytesSent = new LongAdder(); - private final Counter messageSendErrorCounter; - private final Counter messagesSentCounter; - private final Counter bytesSentCounter; - - private final LongAdder messagesReceived = new LongAdder(); - private final LongAdder bytesReceived = new LongAdder(); - private final Counter messagesReceivedCounter; - private final Counter bytesReceivedCounter; - - private final LongAdder totalMessagesSent = new LongAdder(); - private final LongAdder totalMessageSendErrors = new LongAdder(); - private final LongAdder totalMessagesReceived = new LongAdder(); - - private static final long highestTrackableValue = TimeUnit.SECONDS.toMicros(60); - private final Recorder publishLatencyRecorder = new Recorder(highestTrackableValue, 5); - private final Recorder cumulativePublishLatencyRecorder = new Recorder(highestTrackableValue, 5); - private final OpStatsLogger publishLatencyStats; - - private final Recorder publishDelayLatencyRecorder = new Recorder(highestTrackableValue, 5); - private final Recorder cumulativePublishDelayLatencyRecorder = - new Recorder(highestTrackableValue, 5); - private final OpStatsLogger publishDelayLatencyStats; - - private final Recorder endToEndLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5); - private final Recorder endToEndCumulativeLatencyRecorder = - new Recorder(TimeUnit.HOURS.toMicros(12), 5); - private final OpStatsLogger endToEndLatencyStats; - + private final WorkerStats stats; private boolean testCompleted = false; - private boolean consumersArePaused = false; public LocalWorker() { @@ -117,19 +73,7 @@ public LocalWorker() { } public LocalWorker(StatsLogger statsLogger) { - this.statsLogger = statsLogger; - - StatsLogger producerStatsLogger = statsLogger.scope("producer"); - this.messagesSentCounter = producerStatsLogger.getCounter("messages_sent"); - this.messageSendErrorCounter = producerStatsLogger.getCounter("message_send_errors"); - this.bytesSentCounter = producerStatsLogger.getCounter("bytes_sent"); - this.publishDelayLatencyStats = producerStatsLogger.getOpStatsLogger("producer_delay_latency"); - this.publishLatencyStats = producerStatsLogger.getOpStatsLogger("produce_latency"); - - StatsLogger consumerStatsLogger = statsLogger.scope("consumer"); - this.messagesReceivedCounter = consumerStatsLogger.getCounter("messages_recv"); - this.bytesReceivedCounter = consumerStatsLogger.getCounter("bytes_recv"); - this.endToEndLatencyStats = consumerStatsLogger.getOpStatsLogger("e2e_latency"); + stats = new WorkerStats(statsLogger); } @Override @@ -145,7 +89,7 @@ public void initializeDriver(File driverConfigFile) throws IOException { try { benchmarkDriver = (BenchmarkDriver) Class.forName(driverConfiguration.driverClass).newInstance(); - benchmarkDriver.initialize(driverConfigFile, statsLogger); + benchmarkDriver.initialize(driverConfigFile, stats.getStatsLogger()); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException @@ -232,66 +176,24 @@ public void startLoad(ProducerWorkAssignment producerWorkAssignment) { public void probeProducers() throws IOException { producers.forEach( producer -> - producer - .sendAsync(Optional.of("key"), new byte[10]) - .thenRun(() -> totalMessagesSent.increment())); + producer.sendAsync(Optional.of("key"), new byte[10]).thenRun(stats::recordMessageSent)); } - @SuppressWarnings("checkstyle:LineLength") private void submitProducersToExecutor( List producers, KeyDistributor keyDistributor, List payloads) { + MessageProducer messageProducer = new MessageProducer(rateLimiter, stats); + ThreadLocalRandom r = ThreadLocalRandom.current(); + int payloadCount = payloads.size(); executor.submit( () -> { - int payloadCount = payloads.size(); - ThreadLocalRandom r = ThreadLocalRandom.current(); - byte[] firstPayload = payloads.get(0); - try { while (!testCompleted) { producers.forEach( - producer -> { - byte[] payloadData = - payloadCount == 0 ? firstPayload : payloads.get(r.nextInt(payloadCount)); - final long intendedSendTime = rateLimiter.acquire(); - uninterruptibleSleepNs(intendedSendTime); - final long sendTime = System.nanoTime(); - producer - .sendAsync(Optional.ofNullable(keyDistributor.next()), payloadData) - .thenRun( - () -> { - messagesSent.increment(); - totalMessagesSent.increment(); - messagesSentCounter.inc(); - bytesSent.add(payloadData.length); - bytesSentCounter.add(payloadData.length); - - final long latencyMicros = - Math.min( - highestTrackableValue, - TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sendTime)); - publishLatencyRecorder.recordValue(latencyMicros); - cumulativePublishLatencyRecorder.recordValue(latencyMicros); - publishLatencyStats.registerSuccessfulEvent( - latencyMicros, TimeUnit.MICROSECONDS); - - final long sendDelayMicros = - Math.min( - highestTrackableValue, - TimeUnit.NANOSECONDS.toMicros(sendTime - intendedSendTime)); - publishDelayLatencyRecorder.recordValue(sendDelayMicros); - cumulativePublishDelayLatencyRecorder.recordValue(sendDelayMicros); - publishDelayLatencyStats.registerSuccessfulEvent( - sendDelayMicros, TimeUnit.MICROSECONDS); - }) - .exceptionally( - ex -> { - messageSendErrors.increment(); - messageSendErrorCounter.inc(); - totalMessageSendErrors.increment(); - log.warn("Write error on message", ex); - return null; - }); - }); + p -> + messageProducer.sendMessage( + p, + Optional.ofNullable(keyDistributor.next()), + payloads.get(r.nextInt(payloadCount)))); } } catch (Throwable t) { log.error("Got error", t); @@ -310,41 +212,17 @@ public void adjustPublishRate(double publishRate) { @Override public PeriodStats getPeriodStats() { - PeriodStats stats = new PeriodStats(); - - stats.messagesSent = messagesSent.sumThenReset(); - stats.messageSendErrors = messageSendErrors.sumThenReset(); - stats.bytesSent = bytesSent.sumThenReset(); - - stats.messagesReceived = messagesReceived.sumThenReset(); - stats.bytesReceived = bytesReceived.sumThenReset(); - - stats.totalMessagesSent = totalMessagesSent.sum(); - stats.totalMessageSendErrors = totalMessageSendErrors.sum(); - stats.totalMessagesReceived = totalMessagesReceived.sum(); - - stats.publishLatency = publishLatencyRecorder.getIntervalHistogram(); - stats.publishDelayLatency = publishDelayLatencyRecorder.getIntervalHistogram(); - stats.endToEndLatency = endToEndLatencyRecorder.getIntervalHistogram(); - return stats; + return stats.toPeriodStats(); } @Override public CumulativeLatencies getCumulativeLatencies() { - CumulativeLatencies latencies = new CumulativeLatencies(); - latencies.publishLatency = cumulativePublishLatencyRecorder.getIntervalHistogram(); - latencies.publishDelayLatency = cumulativePublishDelayLatencyRecorder.getIntervalHistogram(); - latencies.endToEndLatency = endToEndCumulativeLatencyRecorder.getIntervalHistogram(); - return latencies; + return stats.toCumulativeLatencies(); } @Override public CountersStats getCountersStats() throws IOException { - CountersStats stats = new CountersStats(); - stats.messagesSent = totalMessagesSent.sum(); - stats.messageSendErrors = totalMessageSendErrors.sum(); - stats.messagesReceived = totalMessagesReceived.sum(); - return stats; + return stats.toCountersStats(); } @Override @@ -358,19 +236,9 @@ public void messageReceived(ByteBuffer data, long publishTimestamp) { } public void internalMessageReceived(int size, long publishTimestamp) { - messagesReceived.increment(); - totalMessagesReceived.increment(); - messagesReceivedCounter.inc(); - bytesReceived.add(size); - bytesReceivedCounter.add(size); - long now = System.currentTimeMillis(); long endToEndLatencyMicros = TimeUnit.MILLISECONDS.toMicros(now - publishTimestamp); - if (endToEndLatencyMicros > 0) { - endToEndCumulativeLatencyRecorder.recordValue(endToEndLatencyMicros); - endToEndLatencyRecorder.recordValue(endToEndLatencyMicros); - endToEndLatencyStats.registerSuccessfulEvent(endToEndLatencyMicros, TimeUnit.MICROSECONDS); - } + stats.recordMessageReceived(size, endToEndLatencyMicros); while (consumersArePaused) { try { @@ -395,33 +263,14 @@ public void resumeConsumers() throws IOException { @Override public void resetStats() throws IOException { - publishLatencyRecorder.reset(); - cumulativePublishLatencyRecorder.reset(); - publishDelayLatencyRecorder.reset(); - cumulativePublishDelayLatencyRecorder.reset(); - endToEndLatencyRecorder.reset(); - endToEndCumulativeLatencyRecorder.reset(); + stats.resetLatencies(); } @Override public void stopAll() { testCompleted = true; consumersArePaused = false; - - publishLatencyRecorder.reset(); - cumulativePublishLatencyRecorder.reset(); - publishDelayLatencyRecorder.reset(); - cumulativePublishDelayLatencyRecorder.reset(); - endToEndLatencyRecorder.reset(); - endToEndCumulativeLatencyRecorder.reset(); - - messagesSent.reset(); - messageSendErrors.reset(); - bytesSent.reset(); - messagesReceived.reset(); - bytesReceived.reset(); - totalMessagesSent.reset(); - totalMessagesReceived.reset(); + stats.reset(); try { Thread.sleep(100); @@ -445,6 +294,11 @@ public void stopAll() { } } + @Override + public String id() { + return "local"; + } + @Override public void close() throws Exception { executor.shutdown(); diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/MessageProducer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/MessageProducer.java new file mode 100644 index 000000000..bf1191ef9 --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/MessageProducer.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker; + +import static io.openmessaging.benchmark.utils.UniformRateLimiter.uninterruptibleSleepNs; + +import io.openmessaging.benchmark.driver.BenchmarkProducer; +import io.openmessaging.benchmark.utils.UniformRateLimiter; +import java.util.Optional; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MessageProducer { + + private final WorkerStats stats; + private UniformRateLimiter rateLimiter; + private Supplier nanoClock; + + MessageProducer(UniformRateLimiter rateLimiter, WorkerStats stats) { + this(System::nanoTime, rateLimiter, stats); + } + + MessageProducer(Supplier nanoClock, UniformRateLimiter rateLimiter, WorkerStats stats) { + this.nanoClock = nanoClock; + this.rateLimiter = rateLimiter; + this.stats = stats; + } + + public void sendMessage(BenchmarkProducer producer, Optional key, byte[] payload) { + final long intendedSendTime = rateLimiter.acquire(); + uninterruptibleSleepNs(intendedSendTime); + final long sendTime = nanoClock.get(); + producer + .sendAsync(key, payload) + .thenRun(() -> success(payload.length, intendedSendTime, sendTime)) + .exceptionally(this::failure); + } + + private void success(long payloadLength, long intendedSendTime, long sendTime) { + long nowNs = nanoClock.get(); + stats.recordProducerSuccess(payloadLength, intendedSendTime, sendTime, nowNs); + } + + private Void failure(Throwable t) { + stats.recordProducerFailure(); + log.warn("Write error on message", t); + return null; + } + + private static final Logger log = LoggerFactory.getLogger(MessageProducer.class); +} diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java index 5333f516e..60b0f15d7 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/Worker.java @@ -53,4 +53,6 @@ public interface Worker extends AutoCloseable { void resetStats() throws IOException; void stopAll(); + + String id(); } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java index 05fad840e..f7142cf9f 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerHandler.java @@ -14,19 +14,16 @@ package io.openmessaging.benchmark.worker; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.io.Files; import io.javalin.Context; import io.javalin.Javalin; import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; -import io.openmessaging.benchmark.worker.commands.CumulativeLatencies; -import io.openmessaging.benchmark.worker.commands.PeriodStats; import io.openmessaging.benchmark.worker.commands.ProducerWorkAssignment; import io.openmessaging.benchmark.worker.commands.TopicsInfo; +import io.openmessaging.benchmark.worker.jackson.ObjectMappers; import java.io.File; -import java.nio.ByteBuffer; import java.util.List; import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; @@ -35,25 +32,39 @@ @SuppressWarnings("unchecked") public class WorkerHandler { + public static final String INITIALIZE_DRIVER = "/initialize-driver"; + public static final String CREATE_TOPICS = "/create-topics"; + public static final String CREATE_PRODUCERS = "/create-producers"; + public static final String PROBE_PRODUCERS = "/probe-producers"; + public static final String CREATE_CONSUMERS = "/create-consumers"; + public static final String PAUSE_CONSUMERS = "/pause-consumers"; + public static final String RESUME_CONSUMERS = "/resume-consumers"; + public static final String START_LOAD = "/start-load"; + public static final String ADJUST_PUBLISH_RATE = "/adjust-publish-rate"; + public static final String STOP_ALL = "/stop-all"; + public static final String PERIOD_STATS = "/period-stats"; + public static final String CUMULATIVE_LATENCIES = "/cumulative-latencies"; + public static final String COUNTERS_STATS = "/counters-stats"; + public static final String RESET_STATS = "/reset-stats"; private final Worker localWorker; public WorkerHandler(Javalin app, StatsLogger statsLogger) { this.localWorker = new LocalWorker(statsLogger); - app.post("/initialize-driver", this::handleInitializeDriver); - app.post("/create-topics", this::handleCreateTopics); - app.post("/create-producers", this::handleCreateProducers); - app.post("/probe-producers", this::handleProbeProducers); - app.post("/create-consumers", this::handleCreateConsumers); - app.post("/pause-consumers", this::handlePauseConsumers); - app.post("/resume-consumers", this::handleResumeConsumers); - app.post("/start-load", this::handleStartLoad); - app.post("/adjust-publish-rate", this::handleAdjustPublishRate); - app.post("/stop-all", this::handleStopAll); - app.get("/period-stats", this::handlePeriodStats); - app.get("/cumulative-latencies", this::handleCumulativeLatencies); - app.get("/counters-stats", this::handleCountersStats); - app.post("/reset-stats", this::handleResetStats); + app.post(INITIALIZE_DRIVER, this::handleInitializeDriver); + app.post(CREATE_TOPICS, this::handleCreateTopics); + app.post(CREATE_PRODUCERS, this::handleCreateProducers); + app.post(PROBE_PRODUCERS, this::handleProbeProducers); + app.post(CREATE_CONSUMERS, this::handleCreateConsumers); + app.post(PAUSE_CONSUMERS, this::handlePauseConsumers); + app.post(RESUME_CONSUMERS, this::handleResumeConsumers); + app.post(START_LOAD, this::handleStartLoad); + app.post(ADJUST_PUBLISH_RATE, this::handleAdjustPublishRate); + app.post(STOP_ALL, this::handleStopAll); + app.get(PERIOD_STATS, this::handlePeriodStats); + app.get(CUMULATIVE_LATENCIES, this::handleCumulativeLatencies); + app.get(COUNTERS_STATS, this::handleCountersStats); + app.post(RESET_STATS, this::handleResetStats); } private void handleInitializeDriver(Context ctx) throws Exception { @@ -122,57 +133,11 @@ private void handleStopAll(Context ctx) throws Exception { } private void handlePeriodStats(Context ctx) throws Exception { - PeriodStats stats = localWorker.getPeriodStats(); - - // Serialize histograms - synchronized (histogramSerializationBuffer) { - histogramSerializationBuffer.clear(); - stats.publishLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); - stats.publishLatencyBytes = new byte[histogramSerializationBuffer.position()]; - histogramSerializationBuffer.flip(); - histogramSerializationBuffer.get(stats.publishLatencyBytes); - - histogramSerializationBuffer.clear(); - stats.publishDelayLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); - stats.publishDelayLatencyBytes = new byte[histogramSerializationBuffer.position()]; - histogramSerializationBuffer.flip(); - histogramSerializationBuffer.get(stats.publishDelayLatencyBytes); - - histogramSerializationBuffer.clear(); - stats.endToEndLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); - stats.endToEndLatencyBytes = new byte[histogramSerializationBuffer.position()]; - histogramSerializationBuffer.flip(); - histogramSerializationBuffer.get(stats.endToEndLatencyBytes); - } - - ctx.result(writer.writeValueAsString(stats)); + ctx.result(writer.writeValueAsString(localWorker.getPeriodStats())); } private void handleCumulativeLatencies(Context ctx) throws Exception { - CumulativeLatencies stats = localWorker.getCumulativeLatencies(); - - // Serialize histograms - synchronized (histogramSerializationBuffer) { - histogramSerializationBuffer.clear(); - stats.publishLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); - stats.publishLatencyBytes = new byte[histogramSerializationBuffer.position()]; - histogramSerializationBuffer.flip(); - histogramSerializationBuffer.get(stats.publishLatencyBytes); - - histogramSerializationBuffer.clear(); - stats.publishDelayLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); - stats.publishDelayLatencyBytes = new byte[histogramSerializationBuffer.position()]; - histogramSerializationBuffer.flip(); - histogramSerializationBuffer.get(stats.publishDelayLatencyBytes); - - histogramSerializationBuffer.clear(); - stats.endToEndLatency.encodeIntoCompressedByteBuffer(histogramSerializationBuffer); - stats.endToEndLatencyBytes = new byte[histogramSerializationBuffer.position()]; - histogramSerializationBuffer.flip(); - histogramSerializationBuffer.get(stats.endToEndLatencyBytes); - } - - ctx.result(writer.writeValueAsString(stats)); + ctx.result(writer.writeValueAsString(localWorker.getCumulativeLatencies())); } private void handleCountersStats(Context ctx) throws Exception { @@ -184,16 +149,8 @@ private void handleResetStats(Context ctx) throws Exception { localWorker.resetStats(); } - private final ByteBuffer histogramSerializationBuffer = ByteBuffer.allocate(1024 * 1024); - private static final Logger log = LoggerFactory.getLogger(WorkerHandler.class); - private static final ObjectMapper mapper = - new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - static { - mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE); - } - - private static final ObjectWriter writer = new ObjectMapper().writerWithDefaultPrettyPrinter(); + private static final ObjectMapper mapper = ObjectMappers.DEFAULT.mapper(); + private static final ObjectWriter writer = ObjectMappers.DEFAULT.writer(); } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerStats.java new file mode 100644 index 000000000..68b9d5345 --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/WorkerStats.java @@ -0,0 +1,186 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker; + + +import io.openmessaging.benchmark.worker.commands.CountersStats; +import io.openmessaging.benchmark.worker.commands.CumulativeLatencies; +import io.openmessaging.benchmark.worker.commands.PeriodStats; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; +import org.HdrHistogram.Recorder; +import org.apache.bookkeeper.stats.Counter; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; + +public class WorkerStats { + + private final StatsLogger statsLogger; + + private final OpStatsLogger publishDelayLatencyStats; + + private final Recorder endToEndLatencyRecorder = new Recorder(TimeUnit.HOURS.toMicros(12), 5); + private final Recorder endToEndCumulativeLatencyRecorder = + new Recorder(TimeUnit.HOURS.toMicros(12), 5); + private final OpStatsLogger endToEndLatencyStats; + + private final LongAdder messagesSent = new LongAdder(); + private final LongAdder messageSendErrors = new LongAdder(); + private final LongAdder bytesSent = new LongAdder(); + private final Counter messageSendErrorCounter; + private final Counter messagesSentCounter; + private final Counter bytesSentCounter; + + private final LongAdder messagesReceived = new LongAdder(); + private final LongAdder bytesReceived = new LongAdder(); + private final Counter messagesReceivedCounter; + private final Counter bytesReceivedCounter; + + private final LongAdder totalMessagesSent = new LongAdder(); + private final LongAdder totalMessageSendErrors = new LongAdder(); + private final LongAdder totalMessagesReceived = new LongAdder(); + + private static final long highestTrackableValue = TimeUnit.SECONDS.toMicros(60); + private final Recorder publishLatencyRecorder = new Recorder(highestTrackableValue, 5); + private final Recorder cumulativePublishLatencyRecorder = new Recorder(highestTrackableValue, 5); + private final OpStatsLogger publishLatencyStats; + + private final Recorder publishDelayLatencyRecorder = new Recorder(highestTrackableValue, 5); + private final Recorder cumulativePublishDelayLatencyRecorder = + new Recorder(highestTrackableValue, 5); + + WorkerStats(StatsLogger statsLogger) { + this.statsLogger = statsLogger; + + StatsLogger producerStatsLogger = statsLogger.scope("producer"); + this.messagesSentCounter = producerStatsLogger.getCounter("messages_sent"); + this.messageSendErrorCounter = producerStatsLogger.getCounter("message_send_errors"); + this.bytesSentCounter = producerStatsLogger.getCounter("bytes_sent"); + this.publishDelayLatencyStats = producerStatsLogger.getOpStatsLogger("producer_delay_latency"); + this.publishLatencyStats = producerStatsLogger.getOpStatsLogger("produce_latency"); + + StatsLogger consumerStatsLogger = statsLogger.scope("consumer"); + this.messagesReceivedCounter = consumerStatsLogger.getCounter("messages_recv"); + this.bytesReceivedCounter = consumerStatsLogger.getCounter("bytes_recv"); + this.endToEndLatencyStats = consumerStatsLogger.getOpStatsLogger("e2e_latency"); + } + + public StatsLogger getStatsLogger() { + return statsLogger; + } + + public void recordMessageSent() { + totalMessagesSent.increment(); + } + + public void recordMessageReceived(long payloadLength, long endToEndLatencyMicros) { + messagesReceived.increment(); + totalMessagesReceived.increment(); + messagesReceivedCounter.inc(); + bytesReceived.add(payloadLength); + bytesReceivedCounter.add(payloadLength); + + if (endToEndLatencyMicros > 0) { + endToEndCumulativeLatencyRecorder.recordValue(endToEndLatencyMicros); + endToEndLatencyRecorder.recordValue(endToEndLatencyMicros); + endToEndLatencyStats.registerSuccessfulEvent(endToEndLatencyMicros, TimeUnit.MICROSECONDS); + } + } + + public PeriodStats toPeriodStats() { + PeriodStats stats = new PeriodStats(); + + stats.messagesSent = messagesSent.sumThenReset(); + stats.messageSendErrors = messageSendErrors.sumThenReset(); + stats.bytesSent = bytesSent.sumThenReset(); + + stats.messagesReceived = messagesReceived.sumThenReset(); + stats.bytesReceived = bytesReceived.sumThenReset(); + + stats.totalMessagesSent = totalMessagesSent.sum(); + stats.totalMessageSendErrors = totalMessageSendErrors.sum(); + stats.totalMessagesReceived = totalMessagesReceived.sum(); + + stats.publishLatency = publishLatencyRecorder.getIntervalHistogram(); + stats.publishDelayLatency = publishDelayLatencyRecorder.getIntervalHistogram(); + stats.endToEndLatency = endToEndLatencyRecorder.getIntervalHistogram(); + return stats; + } + + public CumulativeLatencies toCumulativeLatencies() { + CumulativeLatencies latencies = new CumulativeLatencies(); + latencies.publishLatency = cumulativePublishLatencyRecorder.getIntervalHistogram(); + latencies.publishDelayLatency = cumulativePublishDelayLatencyRecorder.getIntervalHistogram(); + latencies.endToEndLatency = endToEndCumulativeLatencyRecorder.getIntervalHistogram(); + return latencies; + } + + public CountersStats toCountersStats() throws IOException { + CountersStats stats = new CountersStats(); + stats.messagesSent = totalMessagesSent.sum(); + stats.messageSendErrors = totalMessageSendErrors.sum(); + stats.messagesReceived = totalMessagesReceived.sum(); + return stats; + } + + public void resetLatencies() { + publishLatencyRecorder.reset(); + cumulativePublishLatencyRecorder.reset(); + publishDelayLatencyRecorder.reset(); + cumulativePublishDelayLatencyRecorder.reset(); + endToEndLatencyRecorder.reset(); + endToEndCumulativeLatencyRecorder.reset(); + } + + public void reset() { + resetLatencies(); + + messagesSent.reset(); + messageSendErrors.reset(); + bytesSent.reset(); + messagesReceived.reset(); + bytesReceived.reset(); + totalMessagesSent.reset(); + totalMessagesReceived.reset(); + } + + public void recordProducerFailure() { + messageSendErrors.increment(); + messageSendErrorCounter.inc(); + totalMessageSendErrors.increment(); + } + + public void recordProducerSuccess( + long payloadLength, long intendedSendTimeNs, long sendTimeNs, long nowNs) { + messagesSent.increment(); + totalMessagesSent.increment(); + messagesSentCounter.inc(); + bytesSent.add(payloadLength); + bytesSentCounter.add(payloadLength); + + final long latencyMicros = + Math.min(highestTrackableValue, TimeUnit.NANOSECONDS.toMicros(nowNs - sendTimeNs)); + publishLatencyRecorder.recordValue(latencyMicros); + cumulativePublishLatencyRecorder.recordValue(latencyMicros); + publishLatencyStats.registerSuccessfulEvent(latencyMicros, TimeUnit.MICROSECONDS); + + final long sendDelayMicros = + Math.min( + highestTrackableValue, TimeUnit.NANOSECONDS.toMicros(sendTimeNs - intendedSendTimeNs)); + publishDelayLatencyRecorder.recordValue(sendDelayMicros); + cumulativePublishDelayLatencyRecorder.recordValue(sendDelayMicros); + publishDelayLatencyStats.registerSuccessfulEvent(sendDelayMicros, TimeUnit.MICROSECONDS); + } +} diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java index a7f935f86..6ddc593e1 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CountersStats.java @@ -17,4 +17,16 @@ public class CountersStats { public long messagesSent; public long messagesReceived; public long messageSendErrors; + + public CountersStats plus(CountersStats toAdd) { + CountersStats result = new CountersStats(); + result.messagesSent += this.messagesSent; + result.messagesReceived += this.messagesReceived; + result.messageSendErrors += this.messageSendErrors; + + result.messagesSent += toAdd.messagesSent; + result.messagesReceived += toAdd.messagesReceived; + result.messageSendErrors += toAdd.messageSendErrors; + return result; + } } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java index e489d9107..e9afea679 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/CumulativeLatencies.java @@ -13,21 +13,28 @@ */ package io.openmessaging.benchmark.worker.commands; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.SECONDS; -import com.fasterxml.jackson.annotation.JsonIgnore; -import java.util.concurrent.TimeUnit; import org.HdrHistogram.Histogram; public class CumulativeLatencies { - @JsonIgnore public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); - public byte[] publishLatencyBytes; + public Histogram publishLatency = new Histogram(SECONDS.toMicros(60), 5); + public Histogram publishDelayLatency = new Histogram(SECONDS.toMicros(60), 5); + public Histogram endToEndLatency = new Histogram(HOURS.toMicros(12), 5); - @JsonIgnore - public Histogram publishDelayLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); + public CumulativeLatencies plus(CumulativeLatencies toAdd) { + CumulativeLatencies result = new CumulativeLatencies(); - public byte[] publishDelayLatencyBytes; + result.publishLatency.add(this.publishLatency); + result.publishDelayLatency.add(this.publishDelayLatency); + result.endToEndLatency.add(this.endToEndLatency); - @JsonIgnore public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5); - public byte[] endToEndLatencyBytes; + result.publishLatency.add(toAdd.publishLatency); + result.publishDelayLatency.add(toAdd.publishDelayLatency); + result.endToEndLatency.add(toAdd.endToEndLatency); + + return result; + } } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java index ec99f75dd..3f06f6925 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/PeriodStats.java @@ -13,9 +13,9 @@ */ package io.openmessaging.benchmark.worker.commands; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.SECONDS; -import com.fasterxml.jackson.annotation.JsonIgnore; -import java.util.concurrent.TimeUnit; import org.HdrHistogram.Histogram; public class PeriodStats { @@ -30,14 +30,37 @@ public class PeriodStats { public long totalMessageSendErrors = 0; public long totalMessagesReceived = 0; - @JsonIgnore public Histogram publishLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); - public byte[] publishLatencyBytes; + public Histogram publishLatency = new Histogram(SECONDS.toMicros(60), 5); + public Histogram publishDelayLatency = new Histogram(SECONDS.toMicros(60), 5); + public Histogram endToEndLatency = new Histogram(HOURS.toMicros(12), 5); - @JsonIgnore - public Histogram publishDelayLatency = new Histogram(TimeUnit.SECONDS.toMicros(60), 5); + public PeriodStats plus(PeriodStats toAdd) { + PeriodStats result = new PeriodStats(); - public byte[] publishDelayLatencyBytes; + result.messagesSent += this.messagesSent; + result.messageSendErrors += this.messageSendErrors; + result.bytesSent += this.bytesSent; + result.messagesReceived += this.messagesReceived; + result.bytesReceived += this.bytesReceived; + result.totalMessagesSent += this.totalMessagesSent; + result.totalMessageSendErrors += this.totalMessageSendErrors; + result.totalMessagesReceived += this.totalMessagesReceived; + result.publishLatency.add(this.publishLatency); + result.publishDelayLatency.add(this.publishDelayLatency); + result.endToEndLatency.add(this.endToEndLatency); - @JsonIgnore public Histogram endToEndLatency = new Histogram(TimeUnit.HOURS.toMicros(12), 5); - public byte[] endToEndLatencyBytes; + result.messagesSent += toAdd.messagesSent; + result.messageSendErrors += toAdd.messageSendErrors; + result.bytesSent += toAdd.bytesSent; + result.messagesReceived += toAdd.messagesReceived; + result.bytesReceived += toAdd.bytesReceived; + result.totalMessagesSent += toAdd.totalMessagesSent; + result.totalMessageSendErrors += toAdd.totalMessageSendErrors; + result.totalMessagesReceived += toAdd.totalMessagesReceived; + result.publishLatency.add(toAdd.publishLatency); + result.publishDelayLatency.add(toAdd.publishDelayLatency); + result.endToEndLatency.add(toAdd.endToEndLatency); + + return result; + } } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java index 50dfe0704..9506ef7cb 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/commands/ProducerWorkAssignment.java @@ -24,4 +24,12 @@ public class ProducerWorkAssignment { public double publishRate; public KeyDistributorType keyDistributorType; + + public ProducerWorkAssignment withPublishRate(double publishRate) { + ProducerWorkAssignment copy = new ProducerWorkAssignment(); + copy.keyDistributorType = this.keyDistributorType; + copy.payloadData = this.payloadData; + copy.publishRate = publishRate; + return copy; + } } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java new file mode 100644 index 000000000..9817a6301 --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker.jackson; + + +import com.fasterxml.jackson.core.JacksonException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.util.ByteBufferBackedOutputStream; +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import org.HdrHistogram.Histogram; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HistogramDeserializer extends StdDeserializer { + + private final ThreadLocal threadBuffer = + ThreadLocal.withInitial(() -> ByteBuffer.allocate(1024 * 1024)); + + public HistogramDeserializer() { + super(Histogram.class); + } + + @Override + public Histogram deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException, JacksonException { + ByteBuffer buffer = threadBuffer.get(); + buffer.clear(); + try (OutputStream os = new ByteBufferBackedOutputStream(buffer)) { + jsonParser.readBinaryValue(os); + buffer.flip(); + // Long.MIN_VALUE used so that Histogram will defer to the value encoded in the histogram + // value. This + // assumes that it is acceptable for the deserialized value we create to share the same + // parameters of the + // source histogram that was serialized. + return Histogram.decodeFromCompressedByteBuffer(buffer, Long.MIN_VALUE); + } catch (Exception e) { + log.error( + "Failed to decode publish delay latency: {}", + ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(buffer))); + throw new RuntimeException(e); + } + } + + private static final Logger log = LoggerFactory.getLogger(HistogramDeserializer.class); +} diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java new file mode 100644 index 000000000..55703bf0e --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker.jackson; + + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.HdrHistogram.Histogram; + +public class HistogramSerializer extends StdSerializer { + + private final ThreadLocal threadBuffer = + ThreadLocal.withInitial(() -> ByteBuffer.allocate(1024 * 1024)); + + public HistogramSerializer() { + super(Histogram.class); + } + + @Override + public void serialize( + Histogram histogram, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException { + ByteBuffer buffer = threadBuffer.get(); + buffer.clear(); + histogram.encodeIntoCompressedByteBuffer(buffer); + byte[] arr = new byte[buffer.position()]; + buffer.flip(); + buffer.get(arr); + jsonGenerator.writeBinary(arr); + } +} diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/ObjectMappers.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/ObjectMappers.java new file mode 100644 index 000000000..680a5dccc --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/ObjectMappers.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker.jackson; + + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.HdrHistogram.Histogram; + +public enum ObjectMappers { + DEFAULT; + + private static final ObjectMapper mapper = + new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + static { + mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE); + SimpleModule module = new SimpleModule(); + module.addSerializer(Histogram.class, new HistogramSerializer()); + module.addDeserializer(Histogram.class, new HistogramDeserializer()); + mapper.registerModule(module); + } + + private static final ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter(); + + public ObjectMapper mapper() { + return mapper; + } + + public ObjectWriter writer() { + return writer; + } +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsembleTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsembleTest.java new file mode 100644 index 000000000..c6f4805a1 --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsembleTest.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class DistributedWorkersEnsembleTest { + + @ParameterizedTest + @MethodSource("producerCountExpectations") + void getNumberOfProducerWorkers(int workerCount, boolean extraConsumers, int expected) { + List workers = mock(List.class); + when(workers.size()).thenReturn(workerCount); + assertThat(DistributedWorkersEnsemble.getNumberOfProducerWorkers(workers, extraConsumers)) + .isEqualTo(expected); + } + + private static Stream producerCountExpectations() { + return Stream.of( + Arguments.of(2, true, 1), + Arguments.of(3, true, 1), + Arguments.of(4, true, 2), + Arguments.of(5, true, 2), + Arguments.of(6, true, 2), + Arguments.of(7, true, 3), + Arguments.of(8, true, 3), + Arguments.of(9, true, 3), + Arguments.of(2, false, 1), + Arguments.of(3, false, 1), + Arguments.of(4, false, 2), + Arguments.of(5, false, 2), + Arguments.of(6, false, 3), + Arguments.of(7, false, 3), + Arguments.of(8, false, 4), + Arguments.of(9, false, 4)); + } +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CountersStatsTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CountersStatsTest.java new file mode 100644 index 000000000..699304de0 --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CountersStatsTest.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker.commands; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class CountersStatsTest { + + @Test + void plus() { + CountersStats one = new CountersStats(); + one.messagesSent = 1; + one.messageSendErrors = 10; + one.messagesReceived = 100; + CountersStats two = new CountersStats(); + two.messagesSent = 2; + two.messageSendErrors = 20; + two.messagesReceived = 200; + + CountersStats result = one.plus(two); + assertThat(result) + .satisfies( + r -> { + assertThat(r.messagesSent).isEqualTo(3); + assertThat(r.messageSendErrors).isEqualTo(30); + assertThat(r.messagesReceived).isEqualTo(300); + }); + } + + @Test + void zeroPlus() { + CountersStats zero = new CountersStats(); + CountersStats two = new CountersStats(); + two.messagesSent = 2; + two.messageSendErrors = 20; + two.messagesReceived = 200; + + CountersStats result = zero.plus(two); + assertThat(result) + .satisfies( + r -> { + assertThat(r.messagesSent).isEqualTo(2); + assertThat(r.messageSendErrors).isEqualTo(20); + assertThat(r.messagesReceived).isEqualTo(200); + }); + } +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CumulativeLatenciesTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CumulativeLatenciesTest.java new file mode 100644 index 000000000..5919739ff --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/CumulativeLatenciesTest.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker.commands; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class CumulativeLatenciesTest { + + @Test + void zeroPlus() { + CumulativeLatencies one = new CumulativeLatencies(); + CumulativeLatencies two = new CumulativeLatencies(); + CumulativeLatencies result = one.plus(two); + assertThat(result) + .satisfies( + r -> { + assertThat(r.publishLatency).isEqualTo(two.publishLatency); + assertThat(r.publishDelayLatency).isEqualTo(two.publishDelayLatency); + assertThat(r.endToEndLatency).isEqualTo(two.endToEndLatency); + }); + } +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/PeriodStatsTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/PeriodStatsTest.java new file mode 100644 index 000000000..01c3e04b5 --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/commands/PeriodStatsTest.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker.commands; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class PeriodStatsTest { + + @Test + void plus() { + PeriodStats one = new PeriodStats(); + one.messagesSent = 1; + one.messageSendErrors = 2; + one.bytesSent = 3; + one.messagesReceived = 4; + one.bytesReceived = 5; + one.totalMessagesSent = 6; + one.totalMessageSendErrors = 7; + one.totalMessagesReceived = 8; + PeriodStats two = new PeriodStats(); + two.messagesSent = 10; + two.messageSendErrors = 20; + two.bytesSent = 30; + two.messagesReceived = 40; + two.bytesReceived = 50; + two.totalMessagesSent = 60; + two.totalMessageSendErrors = 70; + two.totalMessagesReceived = 80; + PeriodStats result = one.plus(two); + assertThat(result) + .satisfies( + r -> { + assertThat(r.messagesSent).isEqualTo(11); + assertThat(r.messageSendErrors).isEqualTo(22); + assertThat(r.bytesSent).isEqualTo(33); + assertThat(r.messagesReceived).isEqualTo(44); + assertThat(r.bytesReceived).isEqualTo(55); + assertThat(r.totalMessagesSent).isEqualTo(66); + assertThat(r.totalMessageSendErrors).isEqualTo(77); + assertThat(r.totalMessagesReceived).isEqualTo(88); + + two.publishLatency.add(one.publishLatency); + two.publishDelayLatency.add(one.publishDelayLatency); + two.endToEndLatency.add(one.endToEndLatency); + + assertThat(r.publishLatency).isEqualTo(two.publishLatency); + assertThat(r.publishDelayLatency).isEqualTo(two.publishDelayLatency); + assertThat(r.endToEndLatency).isEqualTo(two.endToEndLatency); + }); + } + + @Test + void zeroPlus() { + PeriodStats one = new PeriodStats(); + PeriodStats two = new PeriodStats(); + two.messagesSent = 10; + two.messageSendErrors = 20; + two.bytesSent = 30; + two.messagesReceived = 40; + two.bytesReceived = 50; + two.totalMessagesSent = 60; + two.totalMessageSendErrors = 70; + two.totalMessagesReceived = 80; + PeriodStats result = one.plus(two); + assertThat(result) + .satisfies( + r -> { + assertThat(r.messagesSent).isEqualTo(10); + assertThat(r.messageSendErrors).isEqualTo(20); + assertThat(r.bytesSent).isEqualTo(30); + assertThat(r.messagesReceived).isEqualTo(40); + assertThat(r.bytesReceived).isEqualTo(50); + assertThat(r.totalMessagesSent).isEqualTo(60); + assertThat(r.totalMessageSendErrors).isEqualTo(70); + assertThat(r.totalMessagesReceived).isEqualTo(80); + + assertThat(r.publishLatency).isEqualTo(two.publishLatency); + assertThat(r.publishDelayLatency).isEqualTo(two.publishDelayLatency); + assertThat(r.endToEndLatency).isEqualTo(two.endToEndLatency); + }); + } +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java new file mode 100644 index 000000000..b06ef7566 --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.worker.jackson; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import java.io.IOException; +import org.HdrHistogram.Histogram; +import org.junit.jupiter.api.Test; + +class HistogramSerDeTest { + + @Test + void deserialize() throws IOException { + Histogram value = new Histogram(100_000, 3); + value.recordValue(1); + value.recordValue(100); + value.recordValue(10_000); + + ObjectMapper mapper = new ObjectMapper(); + SimpleModule module = new SimpleModule(); + module.addSerializer(Histogram.class, new HistogramSerializer()); + module.addDeserializer(Histogram.class, new HistogramDeserializer()); + mapper.registerModule(module); + + byte[] serialized = mapper.writeValueAsBytes(value); + Histogram deserialized = mapper.readValue(serialized, Histogram.class); + + assertThat(deserialized).isEqualTo(value); + } +} diff --git a/etc/findbugsExclude.xml b/etc/findbugsExclude.xml index d61aa94e5..150f3a86c 100644 --- a/etc/findbugsExclude.xml +++ b/etc/findbugsExclude.xml @@ -106,4 +106,24 @@ + + + + + + + + + + + + + + + + + + + + From be60e244a1fad6e6e81cd5b270cdfab818b455ea Mon Sep 17 00:00:00 2001 From: Elliot West Date: Thu, 6 Oct 2022 18:39:29 +0100 Subject: [PATCH 068/103] [RabbitMQ] Prometheus & Grafana (#330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Documentation * Provision prometheus server * → HCL2 * Prometheus service * Prometheus config * Fix group name * Node exporter and prometheus installation * Grafana access * Grafana docker service * Deploy RMQ standard dashboards * Update README * Fetch git * Grafana datasource * Use templated datasource * Datasource * Correct vars * Spotless * Document credentials * License * Use persistent messages * Include node_exporter dashboard * Describe included dashboards * Document node exporter * Set cluster name for better grouping in dashboards * Revert "Use persistent messages" This reverts commit 6c52f001c6dcffc38da64fae4a6790f5fc67172b. * Give tasks unique names --- driver-rabbitmq/README.md | 31 +- driver-rabbitmq/deploy/deploy.yaml | 274 +++++++++++++++--- .../deploy/provision-rabbitmq-aws.tf | 78 +++-- .../deploy/templates/grafana-datasource.yml | 27 ++ .../deploy/templates/prometheus.service | 13 + .../deploy/templates/prometheus.yml | 45 +++ .../templates/rabbitmq-dashboard.service | 23 ++ driver-rabbitmq/deploy/terraform.tfvars | 10 +- 8 files changed, 430 insertions(+), 71 deletions(-) create mode 100644 driver-rabbitmq/deploy/templates/grafana-datasource.yml create mode 100644 driver-rabbitmq/deploy/templates/prometheus.service create mode 100644 driver-rabbitmq/deploy/templates/prometheus.yml create mode 100644 driver-rabbitmq/deploy/templates/rabbitmq-dashboard.service diff --git a/driver-rabbitmq/README.md b/driver-rabbitmq/README.md index fc8b68d69..9db033e4b 100644 --- a/driver-rabbitmq/README.md +++ b/driver-rabbitmq/README.md @@ -53,10 +53,11 @@ $ terraform apply That will install the following [EC2](https://aws.amazon.com/ec2) instances (plus some other resources, such as a [Virtual Private Cloud](https://aws.amazon.com/vpc/) (VPC)): -| Resource | Description | Count | -|:-------------------|:------------------------------------------------------------|:------| -| RabbitMQ instances | The VMs on which RabbitMQ brokers will run | 3 | -| Client instance | The VM from which the benchmarking suite itself will be run | 1 | +| Resource | Description | Count | +|:--------------------|:------------------------------------------------------------|:------| +| RabbitMQ instances | The VMs on which RabbitMQ brokers will run | 3 | +| Client instance | The VM from which the benchmarking suite itself will be run | 1 | +| Prometheus instance | The VM on which metrics services will be run | 1 | When you run `terraform apply`, you will be prompted to type `yes`. Type `yes` to continue with the installation or anything else to quit. @@ -115,10 +116,26 @@ $ sudo bin/benchmark --drivers driver-rabbotmq/rabbitmq.yaml workloads/1-topic-1 ## Monitoring -The `rabbitmq_management` plugin is installed, and the HTTP endpoint is exposed on all RabbitMQ instances on port -`15672`. This allows access to the management UI which provides some basic status metrics. It should also be possible -to access the management REST API at this endpoint. +### Native + +The `rabbitmq_management` plugin is installed, and the HTTP endpoint is **publicly** exposed on all RabbitMQ instances +on port `15672`. This allows access to the management UI which provides some basic status metrics. It should also be +possible to access the management REST API at this endpoint. Note that the connection is authenticated but not currently encrypted and so passwords will be passed in plain text. Use the `admin` account configured in the [Terraform](deploy/provision-rabbitmq-aws.tf) file to log in. +### Prometheus + +The `rabbitmq_prometheus` plugin is installed and Prometheus is installed on a standalone instance, along with +[Node Exporter](https://github.com/prometheus/node_exporter) on all brokers to allow the collection of system metrics. +Prometheus exposes a public endpoint `http://${prometheus_host}:9090`. See +'[RabbitMQ.com — Monitoring with Prometheus & Grafana](https://www.rabbitmq.com/prometheus.html)' for more information. + +### Grafana + +Grafana and a set of standard dashboards are installed alongside Prometheus. These are exposed on a public endpoint +`http://${prometheus_host}:3000`. Credentials are `admin`/`admin`. Dashboards included: +* [RabbitMQ's standard dashboards](https://grafana.com/rabbitmq) +* [Node Exporter dashboard](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) + diff --git a/driver-rabbitmq/deploy/deploy.yaml b/driver-rabbitmq/deploy/deploy.yaml index ed339af14..a00888a8c 100644 --- a/driver-rabbitmq/deploy/deploy.yaml +++ b/driver-rabbitmq/deploy/deploy.yaml @@ -17,13 +17,13 @@ connection: ssh become: true tasks: - - name: Format disks + - name: RabbitMQ - Format disks filesystem: fstype: xfs dev: '{{ item }}' with_items: - '/dev/nvme1n1' - - name: Mount disks + - name: RabbitMQ - Mount disks mount: path: "{{ item.path }}" src: "{{ item.src }}" @@ -32,7 +32,7 @@ state: mounted with_items: - { path: "/mnt/data", src: "/dev/nvme1n1" } - - name: Set permissions + - name: RabbitMQ - Set filesystem permissions file: path: "/mnt/data" state: touch @@ -42,93 +42,107 @@ hosts: rabbitmq connection: ssh tasks: - - set_fact: + - name: RabbitMQ - Set software versions + set_fact: erlangVersion: 24.3.4.5 rabbitMqVersion: 3.10.7 - - name: Install RPM packages + - name: RabbitMQ - Install RPM packages yum: pkg={{ item }} state=latest with_items: - wget - sysstat - vim - socat - - name: Install Erlang + - name: RabbitMQ - Install Erlang yum: name: https://github.com/rabbitmq/erlang-rpm/releases/download/v{{ erlangVersion }}/erlang-{{ erlangVersion }}-1.el8.x86_64.rpm state: present disable_gpg_check: yes - - name: Install Rabbitmq Server + - name: RabbitMQ - Install RabbitMQ Server yum: name: https://github.com/rabbitmq/rabbitmq-server/releases/download/v{{ rabbitMqVersion }}/rabbitmq-server-{{ rabbitMqVersion }}-1.el8.noarch.rpm state: present disable_gpg_check: yes - - name: Create rabbitmq-env.conf file + - name: RabbitMQ - Create rabbitmq-env.conf file template: src: "templates/rabbitmq-env.conf" dest: "/etc/rabbitmq/rabbitmq-env.conf" - - systemd: + - name: RabbitMQ - Start standalone + systemd: state: started daemon_reload: yes name: "rabbitmq-server" - - name: Install web management + - name: RabbitMQ - Install web management plugin shell: rabbitmq-plugins enable rabbitmq_management - systemd: state: restarted daemon_reload: yes name: "rabbitmq-server" + - name: RabbitMQ - Install Prometheus monitoring plugin + shell: rabbitmq-plugins enable rabbitmq_prometheus + - name: RabbitMQ - Restart standalone + systemd: + state: restarted + daemon_reload: yes + name: "rabbitmq-server" - - name: Clear servers' Erlang cookie + - name: RabbitMQ - Clear Erlang cookie file: path: /var/lib/rabbitmq/.erlang.cookie state: absent - - name: Copy Erlang cookie + - name: RabbitMQ - Copy Erlang cookie copy: src=erlang.cookie dest=/var/lib/rabbitmq/.erlang.cookie owner=rabbitmq group=rabbitmq mode=0400 - - systemd: + - name: RabbitMQ - Restart standalone + systemd: state: restarted daemon_reload: yes name: "rabbitmq-server" tags: [restart-rabbitmq] - - name: RabbitMQ status + - name: RabbitMQ - Check cluster status shell: rabbitmqctl cluster_status register: result - debug: msg: '{{ result.stdout }}' - - name: Rabbit cluster stop slaves + - name: RabbitMQ - Stop cluster followers shell: rabbitmqctl stop_app when: inventory_hostname != hostvars[groups['rabbitmq'][0]].inventory_hostname - - name: Rabbit cluster slaves join master + - name: RabbitMQ - Join followers to leader shell: rabbitmqctl join_cluster rabbit@{{ hostvars[groups['rabbitmq'][0]].private_ip }} when: inventory_hostname != hostvars[groups['rabbitmq'][0]].inventory_hostname - - name: Start RabbitMQ cluster slaves + - name: RabbitMQ - Set cluster name + shell: rabbitmqctl set_cluster_name benchmark_rabbitmq + when: inventory_hostname == hostvars[groups['rabbitmq'][0]].inventory_hostname + + - name: RabbitMQ - Start cluster followers shell: rabbitmqctl start_app when: inventory_hostname != hostvars[groups['rabbitmq'][0]].inventory_hostname - - name: Show RabbitMQ cluster status + - name: RabbitMQ - Show cluster status shell: rabbitmqctl cluster_status register: result - debug: msg: '{{ result.stdout }}' - - name: create admin/admin profile + - name: RabbitMQ - Create admin/admin profile shell: rabbitmqctl add_user admin admin when: inventory_hostname == hostvars[groups['rabbitmq'][0]].inventory_hostname - - name: set admin tag + - name: RabbitMQ - Set admin tag shell: rabbitmqctl set_user_tags admin administrator when: inventory_hostname == hostvars[groups['rabbitmq'][0]].inventory_hostname - - name: set admin permission + - name: RabbitMQ - Set admin permission shell: rabbitmqctl set_permissions -p "/" admin ".*" ".*" ".*" when: inventory_hostname == hostvars[groups['rabbitmq'][0]].inventory_hostname - - name: Create high availability policy + - name: RabbitMQ - Create high availability policy shell: rabbitmqctl set_policy ha-all "^" '{"ha-mode":"exactly","ha-params":2,"ha-sync-mode":"automatic"}' when: inventory_hostname == hostvars[groups['rabbitmq'][0]].inventory_hostname @@ -137,21 +151,87 @@ connection: ssh become: true tasks: - - name: Set up chronyd + - name: Chrony - Configure template: src: "templates/chrony.conf" dest: "/etc/chrony.conf" - - systemd: + - name: Chrony - Restart + systemd: state: restarted daemon_reload: yes name: "chronyd" +- name: Install RabbitMQ Prometheus node exporters + hosts: rabbitmq + connection: ssh + become: true + tasks: + - name: NodeExporter - Set software versions + set_fact: + nodeExporterVersion: 1.2.2 + - name: NodeExporter - Set binary source URL + set_fact: + nodeExporterBinary: + src: "https://github.com/prometheus/node_exporter/releases/download/v{{ nodeExporterVersion }}/node_exporter-{{ nodeExporterVersion }}.linux-amd64.tar.gz" + remote: yes + when: nodeExporterBinary is not defined + - name: NodeExporter - Add node_exporter user + user: + name: node_exporter + shell: /bin/false + system: true + create_home: no + - name: NodeExporter - Download and extract + unarchive: + src: "{{ nodeExporterBinary.src }}" + dest: /tmp + remote_src: "{{ nodeExporterBinary.remote }}" + - name: NodeExporter - Install binary + copy: + src: "/tmp/node_exporter-{{ nodeExporterVersion }}.linux-amd64/node_exporter" + remote_src: yes + dest: /usr/local/bin/ + owner: node_exporter + group: node_exporter + mode: u+x,g+x,o+x + - name: NodeExporter - Create service + blockinfile: + path: /etc/systemd/system/node_exporter.service + block: | + [Unit] + Description=Prometheus Node Exporter + Wants=network-online.target + After=network-online.target + [Service] + User=node_exporter + Group=node_exporter + Type=simple + ExecStart=/usr/local/bin/node_exporter + [Install] + WantedBy=multi-user.target + create: true + - name: NodeExporter - Reload daemon configuration + systemd: + daemon_reload: yes + - name: NodeExporter - Start and enable service + service: + name: node_exporter + state: started + enabled: yes + - name: NodeExporter - Check port 9100 availability + wait_for: + port: 9100 + state: started + timeout: 5 + - name: RabbitMQ benchmarking client setup hosts: client connection: ssh become: true tasks: - - name: Install RPM packages + - name: Benchmark - Tune kernel + shell: tuned-adm profile latency-performance + - name: Benchmark - Install RPM packages yum: pkg={{ item }} state=latest with_items: - wget @@ -159,48 +239,164 @@ - java-11-openjdk-devel - sysstat - vim - - name: Copy benchmark code + - name: Benchmark - Copy and unpack unarchive: src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz dest: /opt tags: [client-code] - - shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark + - name: Benchmark - Install binary + shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark tags: [client-code] - - shell: tuned-adm profile latency-performance - - - template: + - name: Benchmark - Configure worker + template: src: "templates/workers.yaml" dest: "/opt/benchmark/workers.yaml" tags: [client-code] - - - template: + - name: Benchmark - Configure driver + template: src: "templates/rabbitmq.yaml" dest: "/opt/benchmark/driver-rabbitmq/rabbitmq.yaml" tags: [client-code] - - - name: Configure benchmark-worker memory + - name: Benchmark - Configure worker JVM memory lineinfile: dest: /opt/benchmark/bin/benchmark-worker regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB"' tags: [client-code] - - name: Configure benchmark memory + - name: Benchmark - Configure client JVM memory lineinfile: dest: /opt/benchmark/bin/benchmark regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xmx4G"' tags: [client-code] - - name: Install benchmark-worker systemd service + - name: Benchmark - Install systemd service template: src: "templates/benchmark-worker.service" dest: "/etc/systemd/system/benchmark-worker.service" tags: [client-code] - - systemd: + - name: Benchmark - Start worker + systemd: state: restarted daemon_reload: yes name: "benchmark-worker" tags: [client-code] +- name: Prometheus installation + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Prometheus - Set software versions + set_fact: + prometheusVersion: 2.31.1 + - name: Prometheus - Set binary source URL + set_fact: + prometheusBinary: + src: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheusVersion }}/prometheus-{{ prometheusVersion }}.linux-amd64.tar.gz" + remote: yes + - name: Prometheus - Add RHEL yum repo + shell: yum-config-manager --enable rhui-REGION-rhel-server-extras + when: + - ansible_facts['distribution'] == 'RedHat' + - ansible_facts['distribution_major_version'] | int <= 7 + - name: Prometheus - Create install folders + file: path=/opt/prometheus/data state=absent + - file: path=/opt/prometheus/data state=directory + - name: Prometheus - Download and unarchive binary + unarchive: + src: "{{ prometheusBinary.src }}" + remote_src: "{{ prometheusBinary.remote }}" + dest: /opt/prometheus + extra_opts: ["--strip-components=1"] + +- name: Prometheus setup + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Prometheus - Configure systemd + template: + src: "templates/prometheus.service" + dest: "/etc/systemd/system/prometheus.service" + - name: Prometheus - Configure service + template: + src: "templates/prometheus.yml" + dest: "/opt/prometheus/prometheus.yml" + - name: Prometheus - Restart service + systemd: + state: restarted + daemon_reload: yes + name: "prometheus" + +- name: Grafana installation + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Grafana - Configure yum Docker repo + yum_repository: + name: docker + description: repo for docker + baseurl: "https://download.docker.com/linux/centos/{{ ansible_facts['distribution_major_version'] }}/x86_64/stable/" + gpgcheck: no + when: ansible_facts['distribution'] == 'RedHat' + - name: Grafana - Install Docker + yum: + state: latest + pkg: ['docker-ce'] + - name: Grafana - Start docker + service: + name: docker + state: started + enabled: yes + - name: Grafana - Create install folders + file: path=/opt/grafana state=absent + - file: path=/opt/grafana state=directory + - file: path=/repos/rabbitmq-server state=absent + - file: path=/repos/rabbitmq-server state=directory + - file: path=/repos/grafana-dashboard state=absent + - file: path=/repos/grafana-dashboard state=directory + - name: Grafana - Install Git RPM packages + yum: pkg={{ item }} state=latest + with_items: + - git + - name: Grafana - Clone RabbitMQ repository + git: + repo: https://github.com/rabbitmq/rabbitmq-server.git + dest: /repos/rabbitmq-server + clone: yes + update: yes + - name: Grafana - Clone rfmoz/grafana-dashboard repository + git: + repo: https://github.com/rfmoz/grafana-dashboards.git + dest: /repos/grafana-dashboard + clone: yes + update: yes + - name: Grafana - Copy node_exporter dashboard + copy: remote_src=True src=/repos/grafana-dashboard/prometheus/node-exporter-full.json dest=/repos/rabbitmq-server/deps/rabbitmq_prometheus/docker/grafana/dashboards/ + +- name: Grafana setup + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Grafana - Create data folders + file: path=/opt/rabbitmq state=absent + - file: path=/opt/rabbitmq state=directory + - name: Grafana - Configure Prometheus datasource + template: + src: "templates/grafana-datasource.yml" + dest: "/opt/rabbitmq/grafana-datasource.yml" + - name: Grafana - Configure systemd + template: + src: "templates/rabbitmq-dashboard.service" + dest: "/etc/systemd/system/rabbitmq-dashboard.service" + - name: Grafana - Restart Grafana + systemd: + state: restarted + daemon_reload: yes + name: "rabbitmq-dashboard.service" + - name: List host addresses hosts: localhost become: false @@ -211,4 +407,6 @@ - debug: msg: "Benchmark client {{ item }}" with_items: "{{ groups['client'] }}" - + - debug: + msg: "Prometheus servers {{ item }}" + with_items: "{{ groups['prometheus'] }}" diff --git a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf index 72f4fe746..9588610f4 100644 --- a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf +++ b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf @@ -26,7 +26,7 @@ variable "az" {} variable "ami" {} provider "aws" { - region = "${var.region}" + region = var.region } # Create a VPC to launch our instances into @@ -40,27 +40,27 @@ resource "aws_vpc" "benchmark_vpc" { # Create an internet gateway to give our subnet access to the outside world resource "aws_internet_gateway" "default" { - vpc_id = "${aws_vpc.benchmark_vpc.id}" + vpc_id = aws_vpc.benchmark_vpc.id } # Grant the VPC internet access on its main route table resource "aws_route" "internet_access" { - route_table_id = "${aws_vpc.benchmark_vpc.main_route_table_id}" + route_table_id = aws_vpc.benchmark_vpc.main_route_table_id destination_cidr_block = "0.0.0.0/0" - gateway_id = "${aws_internet_gateway.default.id}" + gateway_id = aws_internet_gateway.default.id } # Create a subnet to launch our instances into resource "aws_subnet" "benchmark_subnet" { - vpc_id = "${aws_vpc.benchmark_vpc.id}" + vpc_id = aws_vpc.benchmark_vpc.id cidr_block = "10.0.0.0/24" map_public_ip_on_launch = true - availability_zone = "${var.az}" + availability_zone = var.az } resource "aws_security_group" "benchmark_security_group" { name = "terraform" - vpc_id = "${aws_vpc.benchmark_vpc.id}" + vpc_id = aws_vpc.benchmark_vpc.id # SSH access from anywhere ingress { @@ -78,6 +78,22 @@ resource "aws_security_group" "benchmark_security_group" { cidr_blocks = ["0.0.0.0/0"] } + # Prometheus access + ingress { + from_port = 9090 + to_port = 9090 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + # Grafana access + ingress { + from_port = 3000 + to_port = 3000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + # All ports open within the VPC ingress { from_port = 0 @@ -100,17 +116,17 @@ resource "aws_security_group" "benchmark_security_group" { } resource "aws_key_pair" "auth" { - key_name = "${var.key_name}" - public_key = "${file(var.public_key_path)}" + key_name = var.key_name + public_key = file(var.public_key_path) } resource "aws_instance" "rabbitmq" { - ami = "${var.ami}" - instance_type = "${var.instance_types["rabbitmq"]}" - key_name = "${aws_key_pair.auth.id}" - subnet_id = "${aws_subnet.benchmark_subnet.id}" - vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"] - count = "${var.num_instances["rabbitmq"]}" + ami = var.ami + instance_type = var.instance_types["rabbitmq"] + key_name = aws_key_pair.auth.id + subnet_id = aws_subnet.benchmark_subnet.id + vpc_security_group_ids = [aws_security_group.benchmark_security_group.id] + count = var.num_instances["rabbitmq"] tags = { Name = "rabbitmq_${count.index}" @@ -118,18 +134,36 @@ resource "aws_instance" "rabbitmq" { } resource "aws_instance" "client" { - ami = "${var.ami}" - instance_type = "${var.instance_types["client"]}" - key_name = "${aws_key_pair.auth.id}" - subnet_id = "${aws_subnet.benchmark_subnet.id}" - vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"] - count = "${var.num_instances["client"]}" + ami = var.ami + instance_type = var.instance_types["client"] + key_name = aws_key_pair.auth.id + subnet_id = aws_subnet.benchmark_subnet.id + vpc_security_group_ids = [aws_security_group.benchmark_security_group.id] + count = var.num_instances["client"] tags = { Name = "rabbitmq_client_${count.index}" } } +resource "aws_instance" "prometheus" { + ami = var.ami + instance_type = var.instance_types["prometheus"] + key_name = aws_key_pair.auth.id + subnet_id = aws_subnet.benchmark_subnet.id + vpc_security_group_ids = [ + aws_security_group.benchmark_security_group.id] + count = var.num_instances["prometheus"] + + tags = { + Name = "prometheus_${count.index}" + } +} + +output "prometheus_host" { + value = aws_instance.prometheus.0.public_ip +} + output "client_ssh_host" { - value = "${aws_instance.client.0.public_ip}" + value = aws_instance.client[0].public_ip } diff --git a/driver-rabbitmq/deploy/templates/grafana-datasource.yml b/driver-rabbitmq/deploy/templates/grafana-datasource.yml new file mode 100644 index 000000000..ec981710e --- /dev/null +++ b/driver-rabbitmq/deploy/templates/grafana-datasource.yml @@ -0,0 +1,27 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: 1 + +datasources: +{% for prom in groups['prometheus'] %} + - name: {{ hostvars[prom].inventory_hostname }} + url: http://{{ hostvars[prom].private_ip }}:9090 + type: prometheus + access: proxy + orgId: 1 + isDefault: true + version: 1 + editable: false +{% endfor %} diff --git a/driver-rabbitmq/deploy/templates/prometheus.service b/driver-rabbitmq/deploy/templates/prometheus.service new file mode 100644 index 000000000..598ce16a8 --- /dev/null +++ b/driver-rabbitmq/deploy/templates/prometheus.service @@ -0,0 +1,13 @@ +[Unit] +Description=Prometheus server +After=network.target + +[Service] +WorkingDirectory=/opt/prometheus +ExecStart=/opt/prometheus/prometheus \ + --web.enable-admin-api \ + --storage.tsdb.path=/opt/prometheus/data \ + --config.file=/opt/prometheus/prometheus.yml + +[Install] +WantedBy=multi-user.target diff --git a/driver-rabbitmq/deploy/templates/prometheus.yml b/driver-rabbitmq/deploy/templates/prometheus.yml new file mode 100644 index 000000000..277dfafa8 --- /dev/null +++ b/driver-rabbitmq/deploy/templates/prometheus.yml @@ -0,0 +1,45 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +global: + scrape_interval: 10s # By default, scrape targets every 15 seconds. + evaluation_interval: 10s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + external_labels: + # TODO: replace `` with the right cluster name. E.g. + # + # cluster: test-cluster + cluster: local + +# Load and evaluate rules in these files every 'evaluation_interval' seconds. +# rule_files: + +scrape_configs: + + - job_name: "broker" + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: +{% for broker in groups['rabbitmq'] %} + - {{ hostvars[broker].private_ip }}:15692 +{% endfor %} + + - job_name: "node_metrics" + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: +{% for broker in groups['rabbitmq'] %} + - {{ hostvars[broker].private_ip }}:9100 +{% endfor %} \ No newline at end of file diff --git a/driver-rabbitmq/deploy/templates/rabbitmq-dashboard.service b/driver-rabbitmq/deploy/templates/rabbitmq-dashboard.service new file mode 100644 index 000000000..a22de7045 --- /dev/null +++ b/driver-rabbitmq/deploy/templates/rabbitmq-dashboard.service @@ -0,0 +1,23 @@ +[Unit] +Description=Pulsar Dashboard +After=prometheus.service +Requires=prometheus.service + +[Service] +WorkingDirectory=/opt/grafana +ExecStartPre=/usr/bin/docker pull grafana/grafana:8.3.4 +ExecStart=/usr/bin/docker run \ + --restart=always \ + --name=systemd_rabbitmq_dashboard \ + -p3000:3000 \ + -e GF_INSTALL_PLUGINS="flant-statusmap-panel,grafana-piechart-panel" \ + -v /opt/rabbitmq/grafana-datasource.yml:/etc/grafana/provisioning/datasources/prometheus.yaml \ + -v /repos/rabbitmq-server/deps/rabbitmq_prometheus/docker/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/rabbitmq.yaml \ + -v /repos/rabbitmq-server/deps/rabbitmq_prometheus/docker/grafana/dashboards:/dashboards \ + grafana/grafana:8.3.4 +ExecStop=/usr/bin/docker stop systemd_rabbitmq_dashboard +ExecStopPost=/usr/bin/docker rm -f systemd_rabbitmq_dashboard +ExecReload=/usr/bin/docker restart systemd_rabbitmq_dashboard + +[Install] +WantedBy=multi-user.target diff --git a/driver-rabbitmq/deploy/terraform.tfvars b/driver-rabbitmq/deploy/terraform.tfvars index 79275c107..edacce0de 100644 --- a/driver-rabbitmq/deploy/terraform.tfvars +++ b/driver-rabbitmq/deploy/terraform.tfvars @@ -4,11 +4,13 @@ az = "us-west-2a" ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { - "rabbitmq" = "i3en.6xlarge" - "client" = "m5n.8xlarge" + "rabbitmq" = "i3en.6xlarge" + "client" = "m5n.8xlarge" + "prometheus" = "t2.large" } num_instances = { - "rabbitmq" = 3 - "client" = 4 + "rabbitmq" = 3 + "client" = 4 + "prometheus" = 1 } From c4ad972d395e49439be7100586df6ec9545d3130 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Fri, 7 Oct 2022 01:23:56 +0100 Subject: [PATCH 069/103] Proportional min backlog (#335) --- .../main/java/io/openmessaging/benchmark/Workload.java | 10 ++++++++++ .../io/openmessaging/benchmark/WorkloadGenerator.java | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java index c437b187e..584a606b8 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Workload.java @@ -51,6 +51,16 @@ public class Workload { * has drained all the backlog and it's on par with the producer */ public long consumerBacklogSizeGB = 0; + /** + * The ratio of the backlog that can remain and yet the backlog still be considered empty, and + * thus the workload can complete at the end of the configured duration. In some systems it is not + * feasible for the backlog to be drained fully and thus the workload will run indefinitely. In + * such circumstances, one may be content to achieve a partial drain such as 99% of the backlog. + * The value should be on somewhere between 0.0 and 1.0, where 1.0 indicates that the backlog + * should be fully drained, and 0.0 indicates a best effort, where the workload will complete + * after the specified time irrespective of how much of the backlog has been drained. + */ + public double backlogDrainRatio = 1.0; public int testDurationMinutes; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 721565ad0..791686d0e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -348,7 +348,7 @@ private void createProducers(List topics) throws IOException { // Add the topic multiple times, one for each producer for (int i = 0; i < workload.producersPerTopic; i++) { - topics.forEach(fullListOfTopics::add); + fullListOfTopics.addAll(topics); } Collections.shuffle(fullListOfTopics); @@ -391,7 +391,9 @@ private void buildAndDrainBacklog(int testDurationMinutes) throws IOException { worker.resumeConsumers(); - final long minBacklog = 1000; + long backlogMessageCapacity = requestedBacklogSize / workload.messageSize; + long backlogEmptyLevel = (long) ((1.0 - workload.backlogDrainRatio) * backlogMessageCapacity); + final long minBacklog = Math.max(1000L, backlogEmptyLevel); while (true) { CountersStats stats = worker.getCountersStats(); From 174e907e2a9cd5920b8c58f084eb5a4d650841a3 Mon Sep 17 00:00:00 2001 From: Dave Maughan Date: Fri, 7 Oct 2022 01:55:21 +0100 Subject: [PATCH 070/103] Fix adjustments of the the max sustainable rate (#336) ## Motivation Max-sustainable rate was broken in one of the recent refactors ## Changes Recreate the `MessageProducer` in the worker when the publish rate is adjusted --- .../benchmark/worker/LocalWorker.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index 8a5c57278..b6416427e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -61,7 +61,7 @@ public class LocalWorker implements Worker, ConsumerCallback { private BenchmarkDriver benchmarkDriver = null; private final List producers = new ArrayList<>(); private final List consumers = new ArrayList<>(); - private volatile UniformRateLimiter rateLimiter = new UniformRateLimiter(1.0); + private volatile MessageProducer messageProducer; private final ExecutorService executor = Executors.newCachedThreadPool(new DefaultThreadFactory("local-worker")); private final WorkerStats stats; @@ -74,6 +74,7 @@ public LocalWorker() { public LocalWorker(StatsLogger statsLogger) { stats = new WorkerStats(statsLogger); + updateMessageProducer(1.0); } @Override @@ -149,7 +150,7 @@ public void createConsumers(ConsumerAssignment consumerAssignment) { public void startLoad(ProducerWorkAssignment producerWorkAssignment) { int processors = Runtime.getRuntime().availableProcessors(); - rateLimiter = new UniformRateLimiter(producerWorkAssignment.publishRate); + updateMessageProducer(producerWorkAssignment.publishRate); Map> processorAssignment = new TreeMap<>(); @@ -181,7 +182,6 @@ public void probeProducers() throws IOException { private void submitProducersToExecutor( List producers, KeyDistributor keyDistributor, List payloads) { - MessageProducer messageProducer = new MessageProducer(rateLimiter, stats); ThreadLocalRandom r = ThreadLocalRandom.current(); int payloadCount = payloads.size(); executor.submit( @@ -204,10 +204,14 @@ private void submitProducersToExecutor( @Override public void adjustPublishRate(double publishRate) { if (publishRate < 1.0) { - rateLimiter = new UniformRateLimiter(1.0); + updateMessageProducer(1.0); return; } - rateLimiter = new UniformRateLimiter(publishRate); + updateMessageProducer(publishRate); + } + + private void updateMessageProducer(double publishRate) { + messageProducer = new MessageProducer(new UniformRateLimiter(publishRate), stats); } @Override From 3abea167831f540e59e507dca648c507de980ccd Mon Sep 17 00:00:00 2001 From: Elliot West Date: Fri, 7 Oct 2022 13:16:49 +0100 Subject: [PATCH 071/103] [RabbitMQ] Queue type configuration option (#331) * Make quorum queues optional * Make queue type configurable * Remove folder mistake and use method ref * Supply different driver files --- .../io/openmessaging/benchmark/Benchmark.java | 10 ++-- driver-rabbitmq/README.md | 4 +- driver-rabbitmq/deploy/alicloud/deploy.yaml | 2 +- driver-rabbitmq/deploy/deploy.yaml | 11 ++-- .../{rabbitmq.yaml => rabbitmq-classic.yaml} | 3 +- .../deploy/templates/rabbitmq-quorum.yaml | 26 ++++++++++ driver-rabbitmq/pom.xml | 8 +++ driver-rabbitmq/rabbitmq.yaml | 3 +- .../rabbitmq/RabbitMqBenchmarkDriver.java | 7 +-- .../driver/rabbitmq/RabbitMqConfig.java | 22 +++++++- .../driver/rabbitmq/QueueTypeTest.java | 36 +++++++++++++ .../driver/rabbitmq/RabbitMqConfigTest.java | 50 +++++++++++++++++++ 12 files changed, 160 insertions(+), 22 deletions(-) rename driver-rabbitmq/deploy/templates/{rabbitmq.yaml => rabbitmq-classic.yaml} (94%) create mode 100644 driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml create mode 100644 driver-rabbitmq/src/test/java/io/openmessaging/benchmark/driver/rabbitmq/QueueTypeTest.java create mode 100644 driver-rabbitmq/src/test/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfigTest.java diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java index d18cf90d7..cf95e116e 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/Benchmark.java @@ -141,7 +141,7 @@ public static void main(String[] args) throws Exception { if (arguments.workers != null && !arguments.workers.isEmpty()) { List workers = - arguments.workers.stream().map(w -> new HttpWorkerClient(w)).collect(toList()); + arguments.workers.stream().map(HttpWorkerClient::new).collect(toList()); worker = new DistributedWorkersEnsemble(workers, arguments.extraConsumers); } else { // Use local worker implementation @@ -183,12 +183,8 @@ public static void main(String[] args) throws Exception { driverConfiguration.name, dateFormat.format(new Date())); - log.info("Writing test result into {}/{}", workloadName, fileName); - File folder = new File(workloadName); - if (!folder.mkdirs()) { - log.debug("Unable to create folder {}", folder); - } - writer.writeValue(new File(folder, fileName), result); + log.info("Writing test result into {}", fileName); + writer.writeValue(new File(fileName), result); generator.close(); } catch (Exception e) { diff --git a/driver-rabbitmq/README.md b/driver-rabbitmq/README.md index 9db033e4b..9b2731784 100644 --- a/driver-rabbitmq/README.md +++ b/driver-rabbitmq/README.md @@ -105,13 +105,13 @@ Once you've successfully SSHed into the client host, you can run all [available ```bash $ cd /opt/benchmark -$ sudo bin/benchmark --drivers driver-rabbitmq/rabbitmq.yaml workloads/*.yaml +$ sudo bin/benchmark --drivers driver-rabbitmq/rabbitmq-classic.yaml workloads/*.yaml ``` You can also run specific workloads in the `workloads` folder. Here's an example: ```bash -$ sudo bin/benchmark --drivers driver-rabbotmq/rabbitmq.yaml workloads/1-topic-1-partitions-1kb.yaml +$ sudo bin/benchmark --drivers driver-rabbotmq/rabbitmq-classic.yaml workloads/1-topic-1-partitions-1kb.yaml ``` ## Monitoring diff --git a/driver-rabbitmq/deploy/alicloud/deploy.yaml b/driver-rabbitmq/deploy/alicloud/deploy.yaml index 145df6b92..994f8e1e1 100644 --- a/driver-rabbitmq/deploy/alicloud/deploy.yaml +++ b/driver-rabbitmq/deploy/alicloud/deploy.yaml @@ -197,7 +197,7 @@ mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark - name: Configure service URL lineinfile: - dest: /opt/benchmark/driver-rabbitmq/rabbitmq.yaml + dest: /opt/benchmark/driver-rabbitmq/rabbitmq-classic.yaml regexp: '^brokerAddress\: ' line: 'brokerAddress: rabbitmaster' - template: diff --git a/driver-rabbitmq/deploy/deploy.yaml b/driver-rabbitmq/deploy/deploy.yaml index a00888a8c..cb2b8f5c7 100644 --- a/driver-rabbitmq/deploy/deploy.yaml +++ b/driver-rabbitmq/deploy/deploy.yaml @@ -252,10 +252,15 @@ src: "templates/workers.yaml" dest: "/opt/benchmark/workers.yaml" tags: [client-code] - - name: Benchmark - Configure driver + - name: Benchmark - Configure classic driver template: - src: "templates/rabbitmq.yaml" - dest: "/opt/benchmark/driver-rabbitmq/rabbitmq.yaml" + src: "templates/rabbitmq-classic.yaml" + dest: "/opt/benchmark/driver-rabbitmq/rabbitmq-classic.yaml" + tags: [client-code] + - name: Benchmark - Configure quorum driver + template: + src: "templates/rabbitmq-quorum.yaml" + dest: "/opt/benchmark/driver-rabbitmq/rabbitmq-quorum.yaml" tags: [client-code] - name: Benchmark - Configure worker JVM memory lineinfile: diff --git a/driver-rabbitmq/deploy/templates/rabbitmq.yaml b/driver-rabbitmq/deploy/templates/rabbitmq-classic.yaml similarity index 94% rename from driver-rabbitmq/deploy/templates/rabbitmq.yaml rename to driver-rabbitmq/deploy/templates/rabbitmq-classic.yaml index bf8f8725e..cd5be803e 100644 --- a/driver-rabbitmq/deploy/templates/rabbitmq.yaml +++ b/driver-rabbitmq/deploy/templates/rabbitmq-classic.yaml @@ -22,4 +22,5 @@ amqpUris: {% for pulsar in groups['rabbitmq'] %} - amqp://admin:admin@{{ hostvars[pulsar].private_ip }}:5672 {% endfor %} -messagePersistence: false \ No newline at end of file +messagePersistence: true +queueType: CLASSIC diff --git a/driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml b/driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml new file mode 100644 index 000000000..90711302b --- /dev/null +++ b/driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml @@ -0,0 +1,26 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +name: RabbitMQ +driverClass: io.openmessaging.benchmark.driver.rabbitmq.RabbitMqBenchmarkDriver + +# RabbitMq client specific configurations + +amqpUris: +{% for pulsar in groups['rabbitmq'] %} + - amqp://admin:admin@{{ hostvars[pulsar].private_ip }}:5672 +{% endfor %} +messagePersistence: false +queueType: QUORUM diff --git a/driver-rabbitmq/pom.xml b/driver-rabbitmq/pom.xml index f6fe0fe29..d92646d1d 100644 --- a/driver-rabbitmq/pom.xml +++ b/driver-rabbitmq/pom.xml @@ -39,5 +39,13 @@ io.netty netty-all + + org.assertj + assertj-core + + + org.junit.jupiter + junit-jupiter + diff --git a/driver-rabbitmq/rabbitmq.yaml b/driver-rabbitmq/rabbitmq.yaml index b41078eca..0269a8b30 100644 --- a/driver-rabbitmq/rabbitmq.yaml +++ b/driver-rabbitmq/rabbitmq.yaml @@ -20,4 +20,5 @@ driverClass: io.openmessaging.benchmark.driver.rabbitmq.RabbitMqBenchmarkDriver amqpUris: - amqp://localhost -messagePersistence: false \ No newline at end of file +messagePersistence: false +queueType: CLASSIC diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index d56442205..2ebc4bdcc 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -30,7 +30,6 @@ import java.io.File; import java.io.IOException; import java.net.URI; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -129,11 +128,7 @@ public CompletableFuture createConsumer( channel.exchangeDeclare(exchange, BuiltinExchangeType.FANOUT, true); // Create the queue channel.queueDeclare( - queueName, - true, - false, - false, - Collections.singletonMap("x-queue-type", "quorum")); + queueName, true, false, false, config.queueType.queueOptions()); channel.queueBind(queueName, exchange, ""); future.complete( new RabbitMqBenchmarkConsumer(channel, queueName, consumerCallback)); diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java index 4d2382046..24bec1876 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java @@ -13,13 +13,33 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; +import static io.openmessaging.benchmark.driver.rabbitmq.RabbitMqConfig.QueueType.CLASSIC; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; public class RabbitMqConfig { public List amqpUris = new ArrayList<>(); - public boolean messagePersistence = false; + public QueueType queueType = CLASSIC; + + public enum QueueType { + CLASSIC { + @Override + Map queueOptions() { + return Collections.emptyMap(); + } + }, + QUORUM { + @Override + Map queueOptions() { + return Collections.singletonMap("x-queue-type", "quorum"); + } + }; + + abstract Map queueOptions(); + } } diff --git a/driver-rabbitmq/src/test/java/io/openmessaging/benchmark/driver/rabbitmq/QueueTypeTest.java b/driver-rabbitmq/src/test/java/io/openmessaging/benchmark/driver/rabbitmq/QueueTypeTest.java new file mode 100644 index 000000000..4ff840b60 --- /dev/null +++ b/driver-rabbitmq/src/test/java/io/openmessaging/benchmark/driver/rabbitmq/QueueTypeTest.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.driver.rabbitmq; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class QueueTypeTest { + + @Test + public void classic() { + assertThat(RabbitMqConfig.QueueType.CLASSIC.queueOptions()).isEmpty(); + } + + @Test + public void quorum() { + assertThat(RabbitMqConfig.QueueType.QUORUM.queueOptions()) + .satisfies( + o -> { + assertThat(o).containsEntry("x-queue-type", "quorum"); + assertThat(o).hasSize(1); + }); + } +} diff --git a/driver-rabbitmq/src/test/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfigTest.java b/driver-rabbitmq/src/test/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfigTest.java new file mode 100644 index 000000000..925974c4e --- /dev/null +++ b/driver-rabbitmq/src/test/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfigTest.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.driver.rabbitmq; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +class RabbitMqConfigTest { + + @Test + public void deserialize() throws JsonProcessingException { + String config = + "{\"amqpUris\":[\"amqp://local\"],\"messagePersistence\":true,\"queueType\":\"QUORUM\"}"; + RabbitMqConfig value = new ObjectMapper().readValue(config, RabbitMqConfig.class); + assertThat(value) + .satisfies( + v -> { + assertThat(v.amqpUris).containsOnly("amqp://local"); + assertThat(v.messagePersistence).isTrue(); + assertThat(v.queueType).isEqualTo(RabbitMqConfig.QueueType.QUORUM); + }); + } + + @Test + public void deserializeWithDefaults() throws JsonProcessingException { + String config = "{\"amqpUris\":[\"amqp://local\"]}"; + RabbitMqConfig value = new ObjectMapper().readValue(config, RabbitMqConfig.class); + assertThat(value) + .satisfies( + v -> { + assertThat(v.amqpUris).containsOnly("amqp://local"); + assertThat(v.messagePersistence).isFalse(); + assertThat(v.queueType).isEqualTo(RabbitMqConfig.QueueType.CLASSIC); + }); + } +} From 07895af3cfdf0709c8642cf98b7f1e1ff91daa0a Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 18 Oct 2022 17:02:33 +0100 Subject: [PATCH 072/103] Allow deployment of benchmark code in isolation (#339) --- driver-rabbitmq/deploy/deploy.yaml | 126 +++++++++++++++-------------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/driver-rabbitmq/deploy/deploy.yaml b/driver-rabbitmq/deploy/deploy.yaml index cb2b8f5c7..29682afa1 100644 --- a/driver-rabbitmq/deploy/deploy.yaml +++ b/driver-rabbitmq/deploy/deploy.yaml @@ -224,67 +224,6 @@ state: started timeout: 5 -- name: RabbitMQ benchmarking client setup - hosts: client - connection: ssh - become: true - tasks: - - name: Benchmark - Tune kernel - shell: tuned-adm profile latency-performance - - name: Benchmark - Install RPM packages - yum: pkg={{ item }} state=latest - with_items: - - wget - - java-11-openjdk - - java-11-openjdk-devel - - sysstat - - vim - - name: Benchmark - Copy and unpack - unarchive: - src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz - dest: /opt - tags: [client-code] - - name: Benchmark - Install binary - shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark - tags: [client-code] - - name: Benchmark - Configure worker - template: - src: "templates/workers.yaml" - dest: "/opt/benchmark/workers.yaml" - tags: [client-code] - - name: Benchmark - Configure classic driver - template: - src: "templates/rabbitmq-classic.yaml" - dest: "/opt/benchmark/driver-rabbitmq/rabbitmq-classic.yaml" - tags: [client-code] - - name: Benchmark - Configure quorum driver - template: - src: "templates/rabbitmq-quorum.yaml" - dest: "/opt/benchmark/driver-rabbitmq/rabbitmq-quorum.yaml" - tags: [client-code] - - name: Benchmark - Configure worker JVM memory - lineinfile: - dest: /opt/benchmark/bin/benchmark-worker - regexp: '^JVM_MEM=' - line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB"' - tags: [client-code] - - name: Benchmark - Configure client JVM memory - lineinfile: - dest: /opt/benchmark/bin/benchmark - regexp: '^JVM_MEM=' - line: 'JVM_MEM="-Xmx4G"' - tags: [client-code] - - name: Benchmark - Install systemd service - template: - src: "templates/benchmark-worker.service" - dest: "/etc/systemd/system/benchmark-worker.service" - tags: [client-code] - - name: Benchmark - Start worker - systemd: - state: restarted - daemon_reload: yes - name: "benchmark-worker" - tags: [client-code] - name: Prometheus installation hosts: prometheus @@ -402,6 +341,71 @@ daemon_reload: yes name: "rabbitmq-dashboard.service" +- name: RabbitMQ benchmarking client setup + hosts: client + connection: ssh + become: true + tasks: + - name: Benchmark - Tune kernel + shell: tuned-adm profile latency-performance + - name: Benchmark - Install RPM packages + yum: pkg={{ item }} state=latest + with_items: + - wget + - java-11-openjdk + - java-11-openjdk-devel + - sysstat + - vim + - name: Benchmark - Clean folder + file: path=/opt/benchmark state=absent + tags: [client-code] + - name: Benchmark - Copy and unpack + unarchive: + src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz + dest: /opt + tags: [client-code] + - name: Benchmark - Install binary + shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark + tags: [client-code] + - name: Benchmark - Configure worker + template: + src: "templates/workers.yaml" + dest: "/opt/benchmark/workers.yaml" + tags: [client-code] + - name: Benchmark - Configure classic driver + template: + src: "templates/rabbitmq-classic.yaml" + dest: "/opt/benchmark/driver-rabbitmq/rabbitmq-classic.yaml" + tags: [client-code] + - name: Benchmark - Configure quorum driver + template: + src: "templates/rabbitmq-quorum.yaml" + dest: "/opt/benchmark/driver-rabbitmq/rabbitmq-quorum.yaml" + tags: [client-code] + - name: Benchmark - Configure worker JVM memory + lineinfile: + dest: /opt/benchmark/bin/benchmark-worker + regexp: '^JVM_MEM=' + line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:+ParallelRefProcEnabled -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=12 -XX:ConcGCThreads=12 -XX:+DisableExplicitGC -XX:-ResizePLAB"' + tags: [client-code] + - name: Benchmark - Configure client JVM memory + lineinfile: + dest: /opt/benchmark/bin/benchmark + regexp: '^JVM_MEM=' + line: 'JVM_MEM="-Xmx4G"' + tags: [client-code] + - name: Benchmark - Install systemd service + template: + src: "templates/benchmark-worker.service" + dest: "/etc/systemd/system/benchmark-worker.service" + tags: [client-code] + - name: Benchmark - Start worker + systemd: + state: restarted + daemon_reload: yes + name: "benchmark-worker" + tags: [client-code] + - name: List host addresses hosts: localhost become: false From eb7d28464ecc755af98445203e28ccf363311cd2 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 18 Oct 2022 17:02:51 +0100 Subject: [PATCH 073/103] =?UTF-8?q?=E2=86=92=20AWS,=20+Prometheus=20+Grafa?= =?UTF-8?q?na,=20+doc,=20versions++=20(#340)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install NATS metrics exporters NATS exporter, format data disks disks, monitoring Mount NATS dashboards streaming vs JetStream Typo Start exporter after NATS Correct docker usage Correct working directory Unambiguous name Include NATs, reference STAN Fix scrape port Formatting License --- README.md | 3 +- driver-nats-streaming/README.md | 151 ++++++++ driver-nats-streaming/deploy/ali/ansible.cfg | 3 + .../{ => ali}/provision-nats-streaming-ali.tf | 0 .../deploy/ali/terraform.tfvars | 15 + driver-nats-streaming/deploy/ansible.cfg | 7 +- driver-nats-streaming/deploy/deploy.yaml | 347 ++++++++++++++++-- .../deploy/provision-nats-streaming-aws.tf | 161 ++++++++ .../deploy/templates/chrony.conf | 49 +++ .../deploy/templates/cluster.conf | 3 +- .../deploy/templates/grafana-dashboards.yml | 24 ++ .../deploy/templates/grafana-datasource.yml | 27 ++ .../nats-streaming-dashboard.service | 23 ++ .../templates/nats-streaming-metrics.service | 27 ++ .../templates/nats-streaming-server.service | 2 +- .../deploy/templates/prometheus.service | 13 + .../deploy/templates/prometheus.yml | 45 +++ driver-nats-streaming/deploy/terraform.tfvars | 21 +- driver-nats-streaming/nats-streaming.yaml | 2 +- driver-nats-streaming/pom.xml | 6 +- 20 files changed, 879 insertions(+), 50 deletions(-) create mode 100644 driver-nats-streaming/README.md create mode 100644 driver-nats-streaming/deploy/ali/ansible.cfg rename driver-nats-streaming/deploy/{ => ali}/provision-nats-streaming-ali.tf (100%) create mode 100644 driver-nats-streaming/deploy/ali/terraform.tfvars create mode 100644 driver-nats-streaming/deploy/provision-nats-streaming-aws.tf create mode 100644 driver-nats-streaming/deploy/templates/chrony.conf create mode 100644 driver-nats-streaming/deploy/templates/grafana-dashboards.yml create mode 100644 driver-nats-streaming/deploy/templates/grafana-datasource.yml create mode 100644 driver-nats-streaming/deploy/templates/nats-streaming-dashboard.service create mode 100644 driver-nats-streaming/deploy/templates/nats-streaming-metrics.service create mode 100644 driver-nats-streaming/deploy/templates/prometheus.service create mode 100644 driver-nats-streaming/deploy/templates/prometheus.yml diff --git a/README.md b/README.md index 9a1c83634..c5253fa7d 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ This repository houses user-friendly, cloud-ready benchmarking suites for the fo * [Apache RocketMQ](https://rocketmq.apache.org) * Generic [JMS](https://javaee.github.io/jms-spec/) * [KoP (Kafka-on-Pulsar)](https://github.com/streamnative/kop) -* [NATS Streaming](https://nats.io/) +* [NATS](https://docs.nats.io/) +* [NATS Streaming (STAN)](https://docs.nats.io/legacy/stan) * [NSQ](https://nsq.io) * [Pravega](https://pravega.io/) * [RabbitMQ](https://www.rabbitmq.com/) diff --git a/driver-nats-streaming/README.md b/driver-nats-streaming/README.md new file mode 100644 index 000000000..74685537b --- /dev/null +++ b/driver-nats-streaming/README.md @@ -0,0 +1,151 @@ +# NATS Streaming benchmarks + +> **Warning** +> This module relates to [NATS Streaming / STAN](https://docs.nats.io/legacy/stan) which has been deprecated +> and replaced by [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream). We do not yet have a benchmark +> driver for JetStream. + +This folder houses the assets necessary to run benchmarks for +[NATS Streaming](https://docs.nats.io/legacy/stan). In order to run these benchmarks, you'll need to: + +* [Create the necessary local artifacts](#creating-local-artifacts) +* [Stand up a NATS cluster](#creating-a-nats-cluster-on-amazon-web-services-aws-using-terraform-and-ansible) + on Amazon Web Services (which includes a client host for running the benchmarks) +* [SSH into the client host](#sshing-into-the-client-host) +* [Run the benchmarks from the client host](#running-the-benchmarks-from-the-client-host) + +## Creating local artifacts + +In order to create the local artifacts necessary to run the NATS benchmarks in AWS, you'll need to have +[Maven](https://maven.apache.org/install.html) installed. Once Maven's installed, you can create the necessary +artifacts with a single Maven command: + +```bash +$ git clone https://github.com/openmessaging/benchmark.git +% cd messaging-benchmark +$ mvn install +``` + +## Creating a NATS cluster on Amazon Web Services (AWS) using Terraform and Ansible + +In order to create an NATS cluster on AWS, you'll need to have the following installed: + +* [Terraform](https://terraform.io) +* [The `terraform-inventory` plugin for Terraform](https://github.com/adammck/terraform-inventory) +* [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) + +In addition, you will need to: + +* [Create an AWS account](https://aws.amazon.com/account/) (or use an existing account) +* [Install the `aws` CLI tool](https://aws.amazon.com/cli/) +* [Configure the `aws` CLI tool](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html) + +Once those conditions are in place, you'll need to create an SSH public and private key at `~/.ssh/nats_streaming_aws` +(private) and `~/.ssh/nats_streaming_aws.pub` (public), respectively. + +```bash +$ ssh-keygen -f ~/.ssh/nats_streaming_aws +``` + +When prompted to enter a passphrase, simply hit **Enter** twice. Then, make sure that the keys have been created: + +```bash +$ ls ~/.ssh/nats_streaming_aws* +``` + +With SSH keys in place, you can create the necessary AWS resources using a single Terraform command: + +```bash +$ cd driver-nats-streaming/deploy +$ terraform init +$ terraform apply +``` + +That will install the following [EC2](https://aws.amazon.com/ec2) instances (plus some other resources, such as a +[Virtual Private Cloud](https://aws.amazon.com/vpc/) (VPC)): + +| Resource | Description | Count | +|:--------------------|:------------------------------------------------------------|:------| +| NATS instances | The VMs on which NATS brokers will run | 3 | +| Client instance | The VM from which the benchmarking suite itself will be run | 4 | +| Prometheus instance | The VM on which metrics services will be run | 1 | + +When you run `terraform apply`, you will be prompted to type `yes`. Type `yes` to continue with the installation or +anything else to quit. + +Once the installation is complete, you will see a confirmation message listing the resources that have been installed. + +### Variables + +There's a handful of configurable parameters related to the Terraform deployment that you can alter by modifying the +defaults in the `terraform.tfvars` file. + +| Variable | Description | Default | +|:------------------|:------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------| +| `region` | The AWS region in which the NATS cluster will be deployed | `us-west-2` | +| `az` | The availability zone in which the NATS cluster will be deployed | `us-west-2a` | +| `public_key_path` | The path to the SSH public key that you've generated | `~/.ssh/rabbitmq_aws.pub` | +| `ami` | The [Amazon Machine Image](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) (AWI) to be used by the cluster's machines | [`ami-9fa343e7`](https://access.redhat.com/articles/3135091) | +| `instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (NATS brokers), `c4.8xlarge` (benchmarking clients) | + +> If you modify the `public_key_path`, make sure that you point to the appropriate SSH key path when running the +> [Ansible playbook](#running-the-ansible-playbook). + +### Running the Ansible playbook + +With the appropriate infrastructure in place, you can install and start the RabbitMQ cluster using Ansible with just +one command. Note that a `TFSTATE` environment must point to the folder in which the `tf.state` file is located. + +```bash +$ TF_STATE=. ansible-playbook \ + --user ec2-user \ + --inventory `which terraform-inventory` \ + deploy.yaml +``` + +> If you're using an SSH private key path different from `~/.ssh/nats_streaming_aws`, you can specify that path using +> the `--private-key` flag, for example `--private-key=~/.ssh/my_key`. + +## SSHing into the client host + +In the [output](https://www.terraform.io/intro/getting-started/outputs.html) produced by Terraform, there's a +`client_ssh_host` variable that provides the IP address for the client EC2 host from which benchmarks can be run. +You can SSH into that host using this command: + +```bash +$ ssh -i ~/.ssh/nats_streaming_aws ec2-user@$(terraform output client_ssh_host) +``` + +## Running the benchmarks from the client host + +Once you've successfully SSHed into the client host, you can run all +[available benchmark workloads](../#benchmarking-workloads) like this: + +```bash +$ cd /opt/benchmark +$ sudo bin/benchmark --drivers driver-nats-streaming/nats-streaming.yaml workloads/*.yaml +``` + +You can also run specific workloads in the `workloads` folder. Here's an example: + +```bash +$ sudo bin/benchmark --drivers driver-nats-streaming/nats-streaming.yaml workloads/1-topic-1-partitions-1kb.yaml +``` + +## Monitoring + +### Prometheus + +The [`prometheus-nats-exporter`](https://github.com/nats-io/prometheus-nats-exporter) +service is installed and Prometheus is installed on a standalone instance, along with +[Node Exporter](https://github.com/prometheus/node_exporter) on all brokers to allow the collection of system metrics. +Prometheus exposes a public endpoint `http://${prometheus_host}:9090`. + +### Grafana + +Grafana and a set of standard dashboards are installed alongside Prometheus. These are exposed on a public endpoint +`http://${prometheus_host}:3000`. Credentials are `admin`/`admin`. Dashboards included: +* [NATS dashboards](https://github.com/nats-io/prometheus-nats-exporter/blob/main/walkthrough/README.md) +* [NATS streaming dashboard](https://github.com/nats-io/prometheus-nats-exporter/blob/main/walkthrough/streaming.md) +* [Node Exporter dashboard](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) + diff --git a/driver-nats-streaming/deploy/ali/ansible.cfg b/driver-nats-streaming/deploy/ali/ansible.cfg new file mode 100644 index 000000000..15d972b75 --- /dev/null +++ b/driver-nats-streaming/deploy/ali/ansible.cfg @@ -0,0 +1,3 @@ +[defaults] +host_key_checking=false +private_key_file=alicloud.pem diff --git a/driver-nats-streaming/deploy/provision-nats-streaming-ali.tf b/driver-nats-streaming/deploy/ali/provision-nats-streaming-ali.tf similarity index 100% rename from driver-nats-streaming/deploy/provision-nats-streaming-ali.tf rename to driver-nats-streaming/deploy/ali/provision-nats-streaming-ali.tf diff --git a/driver-nats-streaming/deploy/ali/terraform.tfvars b/driver-nats-streaming/deploy/ali/terraform.tfvars new file mode 100644 index 000000000..1b872e9e8 --- /dev/null +++ b/driver-nats-streaming/deploy/ali/terraform.tfvars @@ -0,0 +1,15 @@ +region = "cn-shenzhen" +availability_zone = "cn-shenzhen-c" +private_key_file = "alicloud.pem" +key_name = "key-pair-from-terraform-nats-streaming" +image_id = "centos_7_04_64_20G_alibase_201701015.vhd" + +instance_types = { + "nats-streaming-server" = "ecs.se1.4xlarge" #4c16g + "client" = "ecs.n4.4xlarge" +} + +num_instances = { + "nats-streaming-server" = 3 + "client" = 2 +} \ No newline at end of file diff --git a/driver-nats-streaming/deploy/ansible.cfg b/driver-nats-streaming/deploy/ansible.cfg index 15d972b75..4679241b7 100644 --- a/driver-nats-streaming/deploy/ansible.cfg +++ b/driver-nats-streaming/deploy/ansible.cfg @@ -1,3 +1,8 @@ [defaults] host_key_checking=false -private_key_file=alicloud.pem +private_key_file=~/.ssh/nats_streaming_aws + +[privilege_escalation] +become=true +become_method='sudo' +become_user='root' \ No newline at end of file diff --git a/driver-nats-streaming/deploy/deploy.yaml b/driver-nats-streaming/deploy/deploy.yaml index 519e6fb57..f0e9cff62 100644 --- a/driver-nats-streaming/deploy/deploy.yaml +++ b/driver-nats-streaming/deploy/deploy.yaml @@ -12,13 +12,38 @@ # limitations under the License. # +- name: Format and mount disks for NATS hosts + hosts: nats + connection: ssh + become: true + tasks: + - name: NATS - Format disks + filesystem: + fstype: xfs + dev: '{{ item }}' + with_items: + - '/dev/nvme1n1' + - name: NATS - Mount disks + mount: + path: "{{ item.path }}" + src: "{{ item.src }}" + fstype: xfs + opts: defaults,noatime,nodiscard + state: mounted + with_items: + - { path: "/mnt/data", src: "/dev/nvme1n1" } + - name: NATS - Set filesystem permissions + file: + path: "/mnt/data" + state: touch + mode: "0777" -- name: Install NATS-Streaming Cluster - hosts: nats-streaming-server +- name: NATS - Install Cluster + hosts: nats connection: ssh tasks: - set_fact: - natsHostUrl: "{{ hostvars[groups['nats-streaming-server'][0]].private_ip }}" + natsHostUrl: "{{ hostvars[groups['nats'][0]].private_ip }}" private_ip: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" - name: Install RPM packages yum: pkg={{ item }} state=latest @@ -28,102 +53,356 @@ - file: path=/opt/nats-streaming state=absent - file: path=/opt/nats-streaming state=directory - - name: Install Nats-streaming Server + - name: NATS - Install server binary unarchive: - src: "https://github.com/nats-io/nats-streaming-server/releases/download/v0.11.2/nats-streaming-server-v0.11.2-linux-amd64.zip" + src: "https://github.com/nats-io/nats-streaming-server/releases/download/v0.25.2/nats-streaming-server-v0.25.2-linux-amd64.zip" remote_src: yes dest: /opt/nats-streaming - - template: + - name: NATS - Apply server configuration + template: src: "templates/cluster.conf" dest: "/opt/nats-streaming/cluster.conf" - - template: + - name: NATS - Configure server systemd service + template: src: "templates/nats-streaming-server.service" dest: "/etc/systemd/system/nats-streaming-server.service" - - - replace: + - name: NATS - Inject cluster URL + replace: dest: /opt/nats-streaming/cluster.conf regexp: 'natsHostUrl' replace: '{{natsHostUrl}}' - - replace: + - name: NATS - Inject cluster private IP + replace: dest: /opt/nats-streaming/cluster.conf regexp: 'localhost' replace: '{{private_ip}}' - - replace: + - name: NATS - Inject cluster leader setting + replace: dest: /opt/nats-streaming/cluster.conf regexp: 'false' replace: 'true' - when: groups['nats-streaming-server'][0] == inventory_hostname - - replace: + when: groups['nats'][0] == inventory_hostname + - name: NATS - Inject cluster hostname + replace: dest: /opt/nats-streaming/cluster.conf regexp: 'routes' replace: '#routes' - when: groups['nats-streaming-server'][0] == inventory_hostname - - - systemd: + when: groups['nats'][0] == inventory_hostname + - name: NATS - Start service + systemd: state: started daemon_reload: yes name: "nats-streaming-server" +- name: Install NATS Prometheus exporters + hosts: nats + connection: ssh + become: true + tasks: + - name: NATSExporter - Configure yum Docker repo + tags: ['nse'] + yum_repository: + name: docker + description: repo for docker + baseurl: "https://download.docker.com/linux/centos/{{ ansible_facts['distribution_major_version'] }}/x86_64/stable/" + gpgcheck: no + when: ansible_facts['distribution'] == 'RedHat' + - name: NATSExporter - Install Docker + tags: ['nse'] + yum: + state: latest + pkg: ['docker-ce'] + - name: NATSExporter - Start docker + tags: ['nse'] + service: + name: docker + state: started + enabled: yes + +- name: Configure NATS Prometheus exporters + hosts: nats + connection: ssh + become: true + tasks: + - file: path=/opt/nats-streaming-metrics state=absent + tags: ['nse'] + - file: path=/opt/nats-streaming-metrics state=directory + tags: ['nse'] + - name: NATSExporter - Configure systemd + tags: ['nse'] + template: + src: "templates/nats-streaming-metrics.service" + dest: "/etc/systemd/system/nats-streaming-metrics.service" + - name: NATSExporter - Restart NATS exporter + tags: ['nse'] + systemd: + state: restarted + daemon_reload: yes + name: "nats-streaming-metrics.service" + +- name: Install Node exporters + hosts: nats + connection: ssh + become: true + tasks: + - name: NodeExporter - Set software versions + set_fact: + nodeExporterVersion: 1.2.2 + - name: NodeExporter - Set binary source URL + set_fact: + nodeExporterBinary: + src: "https://github.com/prometheus/node_exporter/releases/download/v{{ nodeExporterVersion }}/node_exporter-{{ nodeExporterVersion }}.linux-amd64.tar.gz" + remote: yes + when: nodeExporterBinary is not defined + - name: NodeExporter - Add node_exporter user + user: + name: node_exporter + shell: /bin/false + system: true + create_home: no + - name: NodeExporter - Download and extract + unarchive: + src: "{{ nodeExporterBinary.src }}" + dest: /tmp + remote_src: "{{ nodeExporterBinary.remote }}" + - name: NodeExporter - Install binary + copy: + src: "/tmp/node_exporter-{{ nodeExporterVersion }}.linux-amd64/node_exporter" + remote_src: yes + dest: /usr/local/bin/ + owner: node_exporter + group: node_exporter + mode: u+x,g+x,o+x + - name: NodeExporter - Create service + blockinfile: + path: /etc/systemd/system/node_exporter.service + block: | + [Unit] + Description=Prometheus Node Exporter + Wants=network-online.target + After=network-online.target + [Service] + User=node_exporter + Group=node_exporter + Type=simple + ExecStart=/usr/local/bin/node_exporter + [Install] + WantedBy=multi-user.target + create: true + - name: NodeExporter - Reload daemon configuration + systemd: + daemon_reload: yes + - name: NodeExporter - Start and enable service + service: + name: node_exporter + state: started + enabled: yes + - name: NodeExporter - Check port 9100 availability + wait_for: + port: 9100 + state: started + timeout: 5 + +- name: Prometheus installation + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Prometheus - Set software versions + set_fact: + prometheusVersion: 2.31.1 + - name: Prometheus - Set binary source URL + set_fact: + prometheusBinary: + src: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheusVersion }}/prometheus-{{ prometheusVersion }}.linux-amd64.tar.gz" + remote: yes + - name: Prometheus - Add RHEL yum repo + shell: yum-config-manager --enable rhui-REGION-rhel-server-extras + when: + - ansible_facts['distribution'] == 'RedHat' + - ansible_facts['distribution_major_version'] | int <= 7 + - name: Prometheus - Create install folders + file: path=/opt/prometheus/data state=absent + - file: path=/opt/prometheus/data state=directory + - name: Prometheus - Download and unarchive binary + unarchive: + src: "{{ prometheusBinary.src }}" + remote_src: "{{ prometheusBinary.remote }}" + dest: /opt/prometheus + extra_opts: ["--strip-components=1"] -- name: benchmarking client setup +- name: Prometheus setup + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Prometheus - Configure systemd + template: + src: "templates/prometheus.service" + dest: "/etc/systemd/system/prometheus.service" + - name: Prometheus - Configure service + template: + src: "templates/prometheus.yml" + dest: "/opt/prometheus/prometheus.yml" + - name: Prometheus - Restart service + systemd: + state: restarted + daemon_reload: yes + name: "prometheus" + +- name: Grafana installation + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Grafana - Configure yum Docker repo + yum_repository: + name: docker + description: repo for docker + baseurl: "https://download.docker.com/linux/centos/{{ ansible_facts['distribution_major_version'] }}/x86_64/stable/" + gpgcheck: no + when: ansible_facts['distribution'] == 'RedHat' + - name: Grafana - Install Docker + yum: + state: latest + pkg: ['docker-ce'] + - name: Grafana - Start docker + service: + name: docker + state: started + enabled: yes + - name: Grafana - Create install folders + file: path=/opt/grafana state=absent + - file: path=/opt/grafana state=directory + - file: path=/repos/prometheus-nats-exporter state=absent + - file: path=/repos/prometheus-nats-exporter state=directory + - file: path=/repos/grafana-dashboards state=absent + - file: path=/repos/grafana-dashboards state=directory + - name: Grafana - Install Git RPM packages + yum: pkg={{ item }} state=latest + with_items: + - git + - name: Grafana - Clone nats-io/prometheus-nats-exporter repository + git: + repo: https://github.com/nats-io/prometheus-nats-exporter.git + dest: /repos/prometheus-nats-exporter + clone: yes + update: yes + - name: Grafana - Clone rfmoz/grafana-dashboard repository + git: + repo: https://github.com/rfmoz/grafana-dashboards.git + dest: /repos/grafana-dashboards + clone: yes + update: yes + - name: Grafana - Copy node_exporter dashboard + copy: remote_src=True src=/repos/grafana-dashboards/prometheus/node-exporter-full.json dest=/repos/prometheus-nats-exporter/walkthrough/ + +- name: Grafana setup + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Grafana - Create data folders + file: path=/opt/nats-streaming state=absent + - file: path=/opt/nats-streaming state=directory + - name: Grafana - Configure Dashboards + template: + src: "templates/grafana-dashboards.yml" + dest: "/opt/nats-streaming/dashboards.yml" + - name: Grafana - Configure Prometheus datasource + template: + src: "templates/grafana-datasource.yml" + dest: "/opt/nats-streaming/grafana-datasource.yml" + - name: Grafana - Configure systemd + template: + src: "templates/nats-streaming-dashboard.service" + dest: "/etc/systemd/system/nats-streaming-dashboard.service" + - name: Grafana - Restart Grafana + systemd: + state: restarted + daemon_reload: yes + name: "nats-streaming-dashboard.service" + +- name: Chrony setup + hosts: client + connection: ssh + become: true + tasks: + - name: Chrony - Configure + template: + src: "templates/chrony.conf" + dest: "/etc/chrony.conf" + - name: Chrony - Restart + systemd: + state: restarted + daemon_reload: yes + name: "chronyd" + +- name: Benchmarking client setup hosts: client connection: ssh become: true tasks: - set_fact: - natsHostUrl: "{{ hostvars[groups['nats-streaming-server'][0]].private_ip }}" - - name: Install RPM packages + natsHostUrl: "{{ hostvars[groups['nats'][0]].private_ip }}" + - name: Benchmark - Install RPM packages yum: pkg={{ item }} state=latest with_items: - java - - file: path=/opt/benchmark state=absent + - name: Benchmark - Create folders + file: path=/opt/benchmark state=absent - file: path=/opt/benchmark state=directory - - name: Copy benchmark code + - name: Benchmark - Copy code unarchive: src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz dest: /opt - - shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT/* /opt/benchmark - - shell: tuned-adm profile latency-performance - - template: + - name: Benchmark - Install code + shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT/* /opt/benchmark + - name: Benchmark - Tune kernel + shell: tuned-adm profile latency-performance + - name: Benchmark - Configure workers + template: src: "templates/workers.yaml" dest: "/opt/benchmark/workers.yaml" - - name: Get list of driver config files + - name: Benchmark - Get list of driver config files raw: ls -1 /opt/benchmark/driver-nats-streaming/*.yaml register: drivers_list - - - name: Configure natsHost URL + - name: Benchmark - Configure natsHost URL lineinfile: dest: /opt/benchmark/driver-nats-streaming/nats-streaming.yaml regexp: '^natsHostUrl\: ' line: 'natsHostUrl: nats://{{natsHostUrl}}:4222' - - name: Configure memory + - name: Benchmark - Configure worker memory lineinfile: dest: /opt/benchmark/bin/benchmark-worker regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xms12G -Xmx12G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"' - - name: Configure memory + - name: Benchmark - Configure client memory lineinfile: dest: /opt/benchmark/bin/benchmark regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xmx4G"' - - name: Install benchmark systemd service + - name: Benchmark - Install systemd service template: src: "templates/benchmark-worker.service" dest: "/etc/systemd/system/benchmark-worker.service" - - systemd: + - name: Benchmark - Start service + systemd: state: restarted daemon_reload: yes name: "benchmark-worker" - - name: List host addresses hosts: localhost become: false tasks: - debug: - msg: "nats-streaming brokers {{ item }}" - with_items: "{{ groups['nats-streaming-server'] }}" + msg: "NATS Streaming brokers {{ item }}" + with_items: "{{ groups['nats'] }}" - debug: msg: "Benchmark client {{ item }}" with_items: "{{ groups['client'] }}" + - debug: + msg: "Prometheus servers {{ item }}" + with_items: "{{ groups['prometheus'] }}" \ No newline at end of file diff --git a/driver-nats-streaming/deploy/provision-nats-streaming-aws.tf b/driver-nats-streaming/deploy/provision-nats-streaming-aws.tf new file mode 100644 index 000000000..defc2b6c4 --- /dev/null +++ b/driver-nats-streaming/deploy/provision-nats-streaming-aws.tf @@ -0,0 +1,161 @@ +variable "public_key_path" { + description = <` with the right cluster name. E.g. + # + # cluster: test-cluster + cluster: local + +# Load and evaluate rules in these files every 'evaluation_interval' seconds. +# rule_files: + +scrape_configs: + + - job_name: "broker" + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: +{% for broker in groups['nats'] %} + - {{ hostvars[broker].private_ip }}:9090 +{% endfor %} + + - job_name: "node_metrics" + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: +{% for broker in groups['nats'] %} + - {{ hostvars[broker].private_ip }}:9100 +{% endfor %} \ No newline at end of file diff --git a/driver-nats-streaming/deploy/terraform.tfvars b/driver-nats-streaming/deploy/terraform.tfvars index 1b872e9e8..c86fa5450 100644 --- a/driver-nats-streaming/deploy/terraform.tfvars +++ b/driver-nats-streaming/deploy/terraform.tfvars @@ -1,15 +1,16 @@ -region = "cn-shenzhen" -availability_zone = "cn-shenzhen-c" -private_key_file = "alicloud.pem" -key_name = "key-pair-from-terraform-nats-streaming" -image_id = "centos_7_04_64_20G_alibase_201701015.vhd" +public_key_path = "~/.ssh/nats_streaming_aws.pub" +region = "us-west-2" +az = "us-west-2a" +ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { - "nats-streaming-server" = "ecs.se1.4xlarge" #4c16g - "client" = "ecs.n4.4xlarge" + "nats" = "i3en.6xlarge" + "client" = "m5n.8xlarge" + "prometheus" = "t2.large" } num_instances = { - "nats-streaming-server" = 3 - "client" = 2 -} \ No newline at end of file + "nats" = 3 + "client" = 4 + "prometheus" = 1 +} diff --git a/driver-nats-streaming/nats-streaming.yaml b/driver-nats-streaming/nats-streaming.yaml index 94d7f250e..576487933 100644 --- a/driver-nats-streaming/nats-streaming.yaml +++ b/driver-nats-streaming/nats-streaming.yaml @@ -13,7 +13,7 @@ # -name: Nats +name: Nats Streaming driverClass: io.openmessaging.benchmark.driver.natsStreaming.NatsStreamingBenchmarkDriver # RabbitMq client specific configurations diff --git a/driver-nats-streaming/pom.xml b/driver-nats-streaming/pom.xml index a589c5277..3852fd65b 100644 --- a/driver-nats-streaming/pom.xml +++ b/driver-nats-streaming/pom.xml @@ -30,10 +30,14 @@ driver-api ${project.version} + + com.google.guava + guava + io.nats java-nats-streaming - 2.1.0 + 2.2.3 From d7cc6c33df3e9bfb31996496cee36fd1000025be Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 18 Oct 2022 17:07:58 +0100 Subject: [PATCH 074/103] [NATS] Deploy in AWS with observability (#342) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * +Prom +Graf _NodeX +doc +aws Delete js Spotless jetstream → nats JetStream → NATS -streaming -streaming Fix binary path and config Cluster members Name VPC Add routes Fix metrics space * Use 2nd disk Co-authored-by: Matteo Merli --- README.md | 4 +- driver-nats/README.md | 151 +++++++ driver-nats/deploy/alicloud/ansible.cfg | 3 + driver-nats/deploy/alicloud/deploy.yaml | 118 ++++++ .../provision-nats-ali.tf} | 0 driver-nats/deploy/alicloud/terraform.tfvars | 15 + driver-nats/deploy/ansible.cfg | 7 +- driver-nats/deploy/deploy.yaml | 379 ++++++++++++++++-- driver-nats/deploy/provision-nats-aws.tf | 161 ++++++++ .../deploy/templates/benchmark-worker.service | 1 + driver-nats/deploy/templates/chrony.conf | 49 +++ driver-nats/deploy/templates/cluster.conf | 25 +- .../deploy/templates/grafana-dashboards.yml | 24 ++ .../deploy/templates/grafana-datasource.yml | 27 ++ .../deploy/templates/nats-dashboard.service | 23 ++ .../deploy/templates/nats-metrics.service | 28 ++ .../deploy/templates/nats-server.service | 2 +- driver-nats/deploy/templates/nats.yaml | 4 +- .../deploy/templates/prometheus.service | 13 + driver-nats/deploy/templates/prometheus.yml | 45 +++ driver-nats/deploy/terraform.tfvars | 21 +- 21 files changed, 1038 insertions(+), 62 deletions(-) create mode 100644 driver-nats/README.md create mode 100644 driver-nats/deploy/alicloud/ansible.cfg create mode 100644 driver-nats/deploy/alicloud/deploy.yaml rename driver-nats/deploy/{provision-rabbitmq-aws.tf => alicloud/provision-nats-ali.tf} (100%) create mode 100644 driver-nats/deploy/alicloud/terraform.tfvars create mode 100644 driver-nats/deploy/provision-nats-aws.tf create mode 100644 driver-nats/deploy/templates/chrony.conf create mode 100644 driver-nats/deploy/templates/grafana-dashboards.yml create mode 100644 driver-nats/deploy/templates/grafana-datasource.yml create mode 100644 driver-nats/deploy/templates/nats-dashboard.service create mode 100644 driver-nats/deploy/templates/nats-metrics.service create mode 100644 driver-nats/deploy/templates/prometheus.service create mode 100644 driver-nats/deploy/templates/prometheus.yml diff --git a/README.md b/README.md index c5253fa7d..c3addedd3 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ This repository houses user-friendly, cloud-ready benchmarking suites for the fo * [Apache RocketMQ](https://rocketmq.apache.org) * Generic [JMS](https://javaee.github.io/jms-spec/) * [KoP (Kafka-on-Pulsar)](https://github.com/streamnative/kop) -* [NATS](https://docs.nats.io/) -* [NATS Streaming (STAN)](https://docs.nats.io/legacy/stan) +* [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream) +* [NATS Streaming (STAN)](https://docs.nats.io/legacy/stan/intro) * [NSQ](https://nsq.io) * [Pravega](https://pravega.io/) * [RabbitMQ](https://www.rabbitmq.com/) diff --git a/driver-nats/README.md b/driver-nats/README.md new file mode 100644 index 000000000..8beb85ce9 --- /dev/null +++ b/driver-nats/README.md @@ -0,0 +1,151 @@ +# NATS JetStream benchmarks + +> **Warning** +> This module relates to [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream) which supersedes the now +> deprecated [NATS Streaming / STAN](https://docs.nats.io/legacy/stan). If you are interested in STAN benchmarks, +> see the [driver-nats-streaming](../driver-nats-streaming) module. + +This folder houses the assets necessary to run benchmarks for +[NATS JetStream](https://docs.nats.io/nats-concepts/jetstream). In order to run these benchmarks, you'll need to: + +* [Create the necessary local artifacts](#creating-local-artifacts) +* [Stand up a JetStream cluster](#creating-a-jetstream-cluster-on-amazon-web-services-aws-using-terraform-and-ansible) + on Amazon Web Services (which includes a client host for running the benchmarks) +* [SSH into the client host](#sshing-into-the-client-host) +* [Run the benchmarks from the client host](#running-the-benchmarks-from-the-client-host) + +## Creating local artifacts + +In order to create the local artifacts necessary to run the JetStream benchmarks in AWS, you'll need to have +[Maven](https://maven.apache.org/install.html) installed. Once Maven's installed, you can create the necessary +artifacts with a single Maven command: + +```bash +$ git clone https://github.com/openmessaging/benchmark.git +% cd messaging-benchmark +$ mvn install +``` + +## Creating a NATS cluster on Amazon Web Services (AWS) using Terraform and Ansible + +In order to create an NATS cluster on AWS, you'll need to have the following installed: + +* [Terraform](https://terraform.io) +* [The `terraform-inventory` plugin for Terraform](https://github.com/adammck/terraform-inventory) +* [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) + +In addition, you will need to: + +* [Create an AWS account](https://aws.amazon.com/account/) (or use an existing account) +* [Install the `aws` CLI tool](https://aws.amazon.com/cli/) +* [Configure the `aws` CLI tool](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html) + +Once those conditions are in place, you'll need to create an SSH public and private key at `~/.ssh/nats_aws` +(private) and `~/.ssh/nats_aws.pub` (public), respectively. + +```bash +$ ssh-keygen -f ~/.ssh/nats_aws +``` + +When prompted to enter a passphrase, simply hit **Enter** twice. Then, make sure that the keys have been created: + +```bash +$ ls ~/.ssh/nats_aws* +``` + +With SSH keys in place, you can create the necessary AWS resources using a single Terraform command: + +```bash +$ cd driver-nats/deploy +$ terraform init +$ terraform apply +``` + +That will install the following [EC2](https://aws.amazon.com/ec2) instances (plus some other resources, such as a +[Virtual Private Cloud](https://aws.amazon.com/vpc/) (VPC)): + +| Resource | Description | Count | +|:--------------------|:------------------------------------------------------------|:------| +| NATS instances | The VMs on which NATS brokers will run | 3 | +| Client instance | The VM from which the benchmarking suite itself will be run | 4 | +| Prometheus instance | The VM on which metrics services will be run | 1 | + +When you run `terraform apply`, you will be prompted to type `yes`. Type `yes` to continue with the installation or +anything else to quit. + +Once the installation is complete, you will see a confirmation message listing the resources that have been installed. + +### Variables + +There's a handful of configurable parameters related to the Terraform deployment that you can alter by modifying the +defaults in the `terraform.tfvars` file. + +| Variable | Description | Default | +|:------------------|:------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------| +| `region` | The AWS region in which the NATS cluster will be deployed | `us-west-2` | +| `az` | The availability zone in which the NATS cluster will be deployed | `us-west-2a` | +| `public_key_path` | The path to the SSH public key that you've generated | `~/.ssh/rabbitmq_aws.pub` | +| `ami` | The [Amazon Machine Image](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) (AWI) to be used by the cluster's machines | [`ami-9fa343e7`](https://access.redhat.com/articles/3135091) | +| `instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (NATS brokers), `c4.8xlarge` (benchmarking clients) | + +> If you modify the `public_key_path`, make sure that you point to the appropriate SSH key path when running the +> [Ansible playbook](#running-the-ansible-playbook). + +### Running the Ansible playbook + +With the appropriate infrastructure in place, you can install and start the NATS cluster using Ansible with just +one command. Note that a `TFSTATE` environment must point to the folder in which the `tf.state` file is located. + +```bash +$ TF_STATE=. ansible-playbook \ + --user ec2-user \ + --inventory `which terraform-inventory` \ + deploy.yaml +``` + +> If you're using an SSH private key path different from `~/.ssh/nats_aws`, you can specify that path using +> the `--private-key` flag, for example `--private-key=~/.ssh/my_key`. + +## SSHing into the client host + +In the [output](https://www.terraform.io/intro/getting-started/outputs.html) produced by Terraform, there's a +`client_ssh_host` variable that provides the IP address for the client EC2 host from which benchmarks can be run. +You can SSH into that host using this command: + +```bash +$ ssh -i ~/.ssh/nats_aws ec2-user@$(terraform output client_ssh_host) +``` + +## Running the benchmarks from the client host + +Once you've successfully SSHed into the client host, you can run all +[available benchmark workloads](../#benchmarking-workloads) like this: + +```bash +$ cd /opt/benchmark +$ sudo bin/benchmark --drivers driver-nats/nats.yaml workloads/*.yaml +``` + +You can also run specific workloads in the `workloads` folder. Here's an example: + +```bash +$ sudo bin/benchmark --drivers driver-nats/nats.yaml workloads/1-topic-1-partitions-1kb.yaml +``` + +## Monitoring + +### Prometheus + +The [`prometheus-nats-exporter`](https://github.com/nats-io/prometheus-nats-exporter) +service is installed and Prometheus is installed on a standalone instance, along with +[Node Exporter](https://github.com/prometheus/node_exporter) on all brokers to allow the collection of system metrics. +Prometheus exposes a public endpoint `http://${prometheus_host}:9090`. + +### Grafana + +Grafana and a set of standard dashboards are installed alongside Prometheus. These are exposed on a public endpoint +`http://${prometheus_host}:3000`. Credentials are `admin`/`admin`. Dashboards included: + +* [NATS dashboards](https://github.com/nats-io/prometheus-nats-exporter/blob/main/walkthrough/README.md) +* [Node Exporter dashboard](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) + diff --git a/driver-nats/deploy/alicloud/ansible.cfg b/driver-nats/deploy/alicloud/ansible.cfg new file mode 100644 index 000000000..15d972b75 --- /dev/null +++ b/driver-nats/deploy/alicloud/ansible.cfg @@ -0,0 +1,3 @@ +[defaults] +host_key_checking=false +private_key_file=alicloud.pem diff --git a/driver-nats/deploy/alicloud/deploy.yaml b/driver-nats/deploy/alicloud/deploy.yaml new file mode 100644 index 000000000..4cd79a751 --- /dev/null +++ b/driver-nats/deploy/alicloud/deploy.yaml @@ -0,0 +1,118 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +- name: Install NATS Cluster + hosts: nats + connection: ssh + tasks: + - set_fact: + natsHostUrl: "{{ hostvars[groups['nats'][0]].private_ip }}" + - name: Install RPM packages + yum: pkg={{ item }} state=latest + with_items: + - wget + - unzip + + - file: path=/opt/nats state=absent + - file: path=/opt/nats state=directory + - name: Install Nats Server + unarchive: + src: "https://github.com/nats-io/nats-server/releases/download/v2.9.3/nats-server-v2.9.3-amd64.rpm" + remote_src: yes + dest: /opt/nats +# extra_opts: ["--strip-components=1"] + - template: + src: "../templates/cluster.conf" + dest: "/opt/nats/cluster.conf" + - template: + src: "../templates/nats-server.service" + dest: "/etc/systemd/system/nats-server.service" + - replace: + dest: /opt/nats/cluster.conf + regexp: 'natsHostUrl' + replace: '{{natsHostUrl}}' + - replace: + dest: /opt/nats/cluster.conf + regexp: 'localhost' + replace: '{{natsHostUrl}}' + - systemd: + state: started + daemon_reload: yes + name: "nats-server" + + +- name: benchmarking client setup + hosts: client + connection: ssh + become: true + tasks: + - set_fact: + natsHostUrl: "{{ hostvars[groups['nats'][0]].private_ip }}" + - name: Install RPM packages + yum: pkg={{ item }} state=latest + with_items: + - java + - name: Copy benchmark code + unarchive: + src: ../../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz + dest: /opt + - shell: rm -rf /opt/benchmark/ + - shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark + - shell: tuned-adm profile latency-performance + - template: + src: "../templates/workers.yaml" + dest: "/opt/benchmark/workers.yaml" + - name: Get list of driver config files + raw: ls -1 /opt/benchmark/driver-rabbitmq/*.yaml + register: drivers_list + - template: + src: "../templates/nats.yaml" + dest: "/opt/benchmark/driver-nats/nats.yaml" + - name: Configure natsHost URL + lineinfile: + dest: /opt/benchmark/driver-nats/nats.yaml + regexp: '^natsHostUrl\: ' + line: 'natsHostUrl: nats://{{natsHostUrl}}:4222' + - name: Configure memory + lineinfile: + dest: /opt/benchmark/bin/benchmark-worker + regexp: '^JVM_MEM=' + line: 'JVM_MEM="-Xms12G -Xmx12G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"' + - name: Configure memory + lineinfile: + dest: /opt/benchmark/bin/benchmark + regexp: '^JVM_MEM=' + line: 'JVM_MEM="-Xmx4G"' + - name: Install benchmark systemd service + template: + src: "../templates/benchmark-worker.service" + dest: "/etc/systemd/system/benchmark-worker.service" + - systemd: + state: restarted + daemon_reload: yes + name: "benchmark-worker" + + +- name: List host addresses + hosts: localhost + become: false + tasks: + - debug: + msg: "nats brokers {{ item }}" + with_items: "{{ groups['nats'] }}" + - debug: + msg: "Benchmark client {{ item }}" + with_items: "{{ groups['client'] }}" + \ No newline at end of file diff --git a/driver-nats/deploy/provision-rabbitmq-aws.tf b/driver-nats/deploy/alicloud/provision-nats-ali.tf similarity index 100% rename from driver-nats/deploy/provision-rabbitmq-aws.tf rename to driver-nats/deploy/alicloud/provision-nats-ali.tf diff --git a/driver-nats/deploy/alicloud/terraform.tfvars b/driver-nats/deploy/alicloud/terraform.tfvars new file mode 100644 index 000000000..14bbf86ee --- /dev/null +++ b/driver-nats/deploy/alicloud/terraform.tfvars @@ -0,0 +1,15 @@ +region = "cn-shenzhen" +availability_zone = "cn-shenzhen-c" +private_key_file = "alicloud.pem" +key_name = "key-pair-from-terraform-nats" +image_id = "centos_7_04_64_20G_alibase_201701015.vhd" + +instance_types = { + "nats" = "ecs.se1.4xlarge" #4c16g + "client" = "ecs.n4.4xlarge" +} + +num_instances = { + "nats" = 1 + "client" = 2 +} \ No newline at end of file diff --git a/driver-nats/deploy/ansible.cfg b/driver-nats/deploy/ansible.cfg index 15d972b75..a33efb2d9 100644 --- a/driver-nats/deploy/ansible.cfg +++ b/driver-nats/deploy/ansible.cfg @@ -1,3 +1,8 @@ [defaults] host_key_checking=false -private_key_file=alicloud.pem +private_key_file=~/.ssh/nats_aws + +[privilege_escalation] +become=true +become_method='sudo' +become_user='root' \ No newline at end of file diff --git a/driver-nats/deploy/deploy.yaml b/driver-nats/deploy/deploy.yaml index 41ab8e614..3ab205213 100644 --- a/driver-nats/deploy/deploy.yaml +++ b/driver-nats/deploy/deploy.yaml @@ -12,107 +12,404 @@ # limitations under the License. # +- name: Format and mount disks for NATS hosts + hosts: nats + connection: ssh + become: true + tasks: + - name: NATS - Format disks + filesystem: + fstype: xfs + dev: '{{ item }}' + with_items: + - '/dev/nvme2n1' + - name: NATS - Mount disks + mount: + path: "{{ item.path }}" + src: "{{ item.src }}" + fstype: xfs + opts: defaults,noatime,nodiscard + state: mounted + with_items: + - { path: "/mnt/data", src: "/dev/nvme2n1" } + - name: NATS - Set filesystem permissions + file: + path: "/mnt/data" + state: touch + mode: "0777" -- name: Install NATS Cluster +- name: NATS - Install Cluster hosts: nats connection: ssh tasks: - set_fact: natsHostUrl: "{{ hostvars[groups['nats'][0]].private_ip }}" - - name: Install RPM packages + private_ip: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}" + tags: [ broker ] + - name: NATS - Install RPM packages + tags: [ broker ] yum: pkg={{ item }} state=latest with_items: - wget - unzip - - file: path=/opt/nats state=absent + tags: [ broker ] - file: path=/opt/nats state=directory - - name: Install Nats Server - unarchive: - src: "https://github.com/nats-io/gnatsd/releases/download/v1.3.0/gnatsd-v1.3.0-linux-amd64.zip" - remote_src: yes - dest: /opt/nats -# extra_opts: ["--strip-components=1"] - - template: - src: "templates/cluster.conf" + tags: [ broker ] + - name: NATS - Install server + tags: [ broker ] + yum: + name: "https://github.com/nats-io/nats-server/releases/download/v2.9.3/nats-server-v2.9.3-amd64.rpm" + state: present + disable_gpg_check: yes + - name: NATS - Apply server configuration + tags: [ broker ] + template: + src: "../../driver-nats/deploy/templates/cluster.conf" dest: "/opt/nats/cluster.conf" - - template: + - name: NATS - Configure server systemd service + tags: [ broker ] + template: src: "templates/nats-server.service" dest: "/etc/systemd/system/nats-server.service" - - replace: - dest: /opt/nats/cluster.conf - regexp: 'natsHostUrl' - replace: '{{natsHostUrl}}' - - replace: - dest: /opt/nats/cluster.conf - regexp: 'localhost' - replace: '{{natsHostUrl}}' - - systemd: + - name: NATS - Start service + tags: [ broker ] + systemd: state: started daemon_reload: yes name: "nats-server" +- name: Install NATS Prometheus exporters + hosts: nats + connection: ssh + become: true + tasks: + - name: NATSExporter - Configure yum Docker repo + tags: ['nse'] + yum_repository: + name: docker + description: repo for docker + baseurl: "https://download.docker.com/linux/centos/{{ ansible_facts['distribution_major_version'] }}/x86_64/stable/" + gpgcheck: no + when: ansible_facts['distribution'] == 'RedHat' + - name: NATSExporter - Install Docker + tags: ['nse'] + yum: + state: latest + pkg: ['docker-ce'] + - name: NATSExporter - Start docker + tags: ['nse'] + service: + name: docker + state: started + enabled: yes -- name: benchmarking client setup +- name: Configure NATS Prometheus exporters + hosts: nats + connection: ssh + become: true + tasks: + - file: path=/opt/nats-metrics state=absent + tags: ['nse'] + - file: path=/opt/nats-metrics state=directory + tags: ['nse'] + - name: NATSExporter - Configure systemd + tags: ['nse'] + template: + src: "templates/nats-metrics.service" + dest: "/etc/systemd/system/nats-metrics.service" + - name: NATSExporter - Restart NATS exporter + tags: ['nse'] + systemd: + state: restarted + daemon_reload: yes + name: "nats-metrics.service" + +- name: Install Node exporters + hosts: nats + connection: ssh + become: true + tasks: + - name: NodeExporter - Set software versions + set_fact: + nodeExporterVersion: 1.2.2 + - name: NodeExporter - Set binary source URL + set_fact: + nodeExporterBinary: + src: "https://github.com/prometheus/node_exporter/releases/download/v{{ nodeExporterVersion }}/node_exporter-{{ nodeExporterVersion }}.linux-amd64.tar.gz" + remote: yes + when: nodeExporterBinary is not defined + - name: NodeExporter - Add node_exporter user + user: + name: node_exporter + shell: /bin/false + system: true + create_home: no + - name: NodeExporter - Download and extract + unarchive: + src: "{{ nodeExporterBinary.src }}" + dest: /tmp + remote_src: "{{ nodeExporterBinary.remote }}" + - name: NodeExporter - Install binary + copy: + src: "/tmp/node_exporter-{{ nodeExporterVersion }}.linux-amd64/node_exporter" + remote_src: yes + dest: /usr/local/bin/ + owner: node_exporter + group: node_exporter + mode: u+x,g+x,o+x + - name: NodeExporter - Create service + blockinfile: + path: /etc/systemd/system/node_exporter.service + block: | + [Unit] + Description=Prometheus Node Exporter + Wants=network-online.target + After=network-online.target + [Service] + User=node_exporter + Group=node_exporter + Type=simple + ExecStart=/usr/local/bin/node_exporter + [Install] + WantedBy=multi-user.target + create: true + - name: NodeExporter - Reload daemon configuration + systemd: + daemon_reload: yes + - name: NodeExporter - Start and enable service + service: + name: node_exporter + state: started + enabled: yes + - name: NodeExporter - Check port 9100 availability + wait_for: + port: 9100 + state: started + timeout: 5 + +- name: Prometheus installation + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Prometheus - Set software versions + set_fact: + prometheusVersion: 2.31.1 + - name: Prometheus - Set binary source URL + set_fact: + prometheusBinary: + src: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheusVersion }}/prometheus-{{ prometheusVersion }}.linux-amd64.tar.gz" + remote: yes + - name: Prometheus - Add RHEL yum repo + shell: yum-config-manager --enable rhui-REGION-rhel-server-extras + when: + - ansible_facts['distribution'] == 'RedHat' + - ansible_facts['distribution_major_version'] | int <= 7 + - name: Prometheus - Create install folders + file: path=/opt/prometheus/data state=absent + - file: path=/opt/prometheus/data state=directory + - name: Prometheus - Download and unarchive binary + unarchive: + src: "{{ prometheusBinary.src }}" + remote_src: "{{ prometheusBinary.remote }}" + dest: /opt/prometheus + extra_opts: ["--strip-components=1"] + +- name: Prometheus setup + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Prometheus - Configure systemd + template: + src: "templates/prometheus.service" + dest: "/etc/systemd/system/prometheus.service" + - name: Prometheus - Configure service + template: + src: "templates/prometheus.yml" + dest: "/opt/prometheus/prometheus.yml" + - name: Prometheus - Restart service + systemd: + state: restarted + daemon_reload: yes + name: "prometheus" + +- name: Grafana installation + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Grafana - Configure yum Docker repo + tags: ['grafins'] + yum_repository: + name: docker + description: repo for docker + baseurl: "https://download.docker.com/linux/centos/{{ ansible_facts['distribution_major_version'] }}/x86_64/stable/" + gpgcheck: no + when: ansible_facts['distribution'] == 'RedHat' + - name: Grafana - Install Docker + tags: ['grafins'] + yum: + state: latest + pkg: ['docker-ce'] + - name: Grafana - Start docker + tags: ['grafins'] + service: + name: docker + state: started + enabled: yes + - name: Grafana - Create install folders + tags: ['grafins'] + file: path=/opt/grafana state=absent + - file: path=/opt/grafana state=directory + tags: ['grafins'] + - file: path=/repos/prometheus-nats-exporter state=absent + tags: ['grafins'] + - file: path=/repos/prometheus-nats-exporter state=directory + tags: ['grafins'] + - file: path=/repos/grafana-dashboards state=absent + tags: ['grafins'] + - file: path=/repos/grafana-dashboards state=directory + tags: ['grafins'] + - name: Grafana - Install Git RPM packages + tags: ['grafins'] + yum: pkg={{ item }} state=latest + with_items: + - git + - name: Grafana - Clone nats-io/prometheus-nats-exporter repository + tags: ['grafins'] + git: + repo: https://github.com/nats-io/prometheus-nats-exporter.git + dest: /repos/prometheus-nats-exporter + clone: yes + update: yes + - name: Grafana - Clone rfmoz/grafana-dashboard repository + tags: ['grafins'] + git: + repo: https://github.com/rfmoz/grafana-dashboards.git + dest: /repos/grafana-dashboards + clone: yes + update: yes + - name: Grafana - Copy node_exporter dashboard + tags: ['grafins'] + copy: remote_src=True src=/repos/grafana-dashboards/prometheus/node-exporter-full.json dest=/repos/prometheus-nats-exporter/walkthrough/ + +- name: Grafana setup + hosts: prometheus + connection: ssh + become: true + tasks: + - name: Grafana - Update datasource + tags: ['grafconf'] + shell: sed -i 's/\${DS__NATS-PROMETHEUS}/NATS-Prometheus/g' /repos/prometheus-nats-exporter/walkthrough/*.json + - name: Grafana - Create data folders + tags: ['grafconf'] + file: path=/opt/nats state=absent + - file: path=/opt/nats state=directory + tags: ['grafconf'] + - name: Grafana - Configure Dashboards + tags: ['grafconf'] + template: + src: "templates/grafana-dashboards.yml" + dest: "/opt/nats/dashboards.yml" + - name: Grafana - Configure Prometheus datasource + tags: ['grafconf'] + template: + src: "templates/grafana-datasource.yml" + dest: "/opt/nats/grafana-datasource.yml" + - name: Grafana - Configure systemd + tags: ['grafconf'] + template: + src: "../../driver-nats/deploy/templates/nats-dashboard.service" + dest: "/etc/systemd/system/nats-dashboard.service" + - name: Grafana - Restart Grafana + tags: ['grafconf'] + systemd: + state: restarted + daemon_reload: yes + name: "nats-dashboard.service" + +- name: Chrony setup + hosts: client + connection: ssh + become: true + tasks: + - name: Chrony - Configure + template: + src: "templates/chrony.conf" + dest: "/etc/chrony.conf" + - name: Chrony - Restart + systemd: + state: restarted + daemon_reload: yes + name: "chronyd" + +- name: Benchmarking client setup hosts: client connection: ssh become: true tasks: - set_fact: natsHostUrl: "{{ hostvars[groups['nats'][0]].private_ip }}" - - name: Install RPM packages + - name: Benchmark - Install RPM packages yum: pkg={{ item }} state=latest with_items: - java - - name: Copy benchmark code + - name: Benchmark - Create folders + file: path=/opt/benchmark state=absent + - file: path=/opt/benchmark state=directory + - name: Benchmark - Copy code unarchive: src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz dest: /opt - - shell: rm -rf /opt/benchmark/ - - shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT /opt/benchmark - - shell: tuned-adm profile latency-performance - - template: - src: "templates/workers.yaml" + - name: Benchmark - Install code + shell: mv /opt/openmessaging-benchmark-0.0.1-SNAPSHOT/* /opt/benchmark + - name: Benchmark - Tune kernel + shell: tuned-adm profile latency-performance + - name: Benchmark - Configure workers + template: + src: "../../driver-nats/deploy/templates/workers.yaml" dest: "/opt/benchmark/workers.yaml" - - name: Get list of driver config files - raw: ls -1 /opt/benchmark/driver-rabbitmq/*.yaml + - name: Benchmark - Get list of driver config files + raw: ls -1 /opt/benchmark/driver-nats/*.yaml register: drivers_list - - template: - src: "templates/nats.yaml" - dest: "/opt/benchmark/driver-nats/nats.yaml" - - name: Configure natsHost URL + - name: Benchmark - Configure natsHost URL lineinfile: dest: /opt/benchmark/driver-nats/nats.yaml regexp: '^natsHostUrl\: ' line: 'natsHostUrl: nats://{{natsHostUrl}}:4222' - - name: Configure memory + - name: Benchmark - Configure worker memory lineinfile: dest: /opt/benchmark/bin/benchmark-worker regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xms12G -Xmx12G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"' - - name: Configure memory + - name: Benchmark - Configure client memory lineinfile: dest: /opt/benchmark/bin/benchmark regexp: '^JVM_MEM=' line: 'JVM_MEM="-Xmx4G"' - - name: Install benchmark systemd service + - name: Benchmark - Install systemd service template: src: "templates/benchmark-worker.service" dest: "/etc/systemd/system/benchmark-worker.service" - - systemd: + - name: Benchmark - Start service + systemd: state: restarted daemon_reload: yes name: "benchmark-worker" - - name: List host addresses hosts: localhost become: false tasks: - debug: - msg: "nats brokers {{ item }}" + msg: "Nats brokers {{ item }}" with_items: "{{ groups['nats'] }}" - debug: msg: "Benchmark client {{ item }}" with_items: "{{ groups['client'] }}" + - debug: + msg: "Prometheus servers {{ item }}" + with_items: "{{ groups['prometheus'] }}" \ No newline at end of file diff --git a/driver-nats/deploy/provision-nats-aws.tf b/driver-nats/deploy/provision-nats-aws.tf new file mode 100644 index 000000000..76d51c09f --- /dev/null +++ b/driver-nats/deploy/provision-nats-aws.tf @@ -0,0 +1,161 @@ +variable "public_key_path" { + description = <` with the right cluster name. E.g. + # + # cluster: test-cluster + cluster: local + +# Load and evaluate rules in these files every 'evaluation_interval' seconds. +# rule_files: + +scrape_configs: + + - job_name: "broker" + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: +{% for broker in groups['nats'] %} + - {{ hostvars[broker].private_ip }}:9090 +{% endfor %} + + - job_name: "node_metrics" + honor_labels: true # don't overwrite job & instance labels + static_configs: + - targets: +{% for broker in groups['nats'] %} + - {{ hostvars[broker].private_ip }}:9100 +{% endfor %} \ No newline at end of file diff --git a/driver-nats/deploy/terraform.tfvars b/driver-nats/deploy/terraform.tfvars index 14bbf86ee..c82e0db92 100644 --- a/driver-nats/deploy/terraform.tfvars +++ b/driver-nats/deploy/terraform.tfvars @@ -1,15 +1,16 @@ -region = "cn-shenzhen" -availability_zone = "cn-shenzhen-c" -private_key_file = "alicloud.pem" -key_name = "key-pair-from-terraform-nats" -image_id = "centos_7_04_64_20G_alibase_201701015.vhd" +public_key_path = "~/.ssh/nats_aws.pub" +region = "us-west-2" +az = "us-west-2a" +ami = "ami-08970fb2e5767e3b8" // RHEL-8 instance_types = { - "nats" = "ecs.se1.4xlarge" #4c16g - "client" = "ecs.n4.4xlarge" + "nats" = "i3en.6xlarge" + "client" = "m5n.8xlarge" + "prometheus" = "t2.large" } num_instances = { - "nats" = 1 - "client" = 2 -} \ No newline at end of file + "nats" = 3 + "client" = 4 + "prometheus" = 1 +} From 7175eca8e181f5366fba1470f4720141697c06b1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 24 Oct 2022 19:57:58 +0300 Subject: [PATCH 075/103] [fix][artemis] Fix Artemis asynchronous send and acknowledgement configuration (#343) * [fix][artemis] Fix Artemis confirmation windows size configuration - set to 1MB which is a more sensible default value - see https://activemq.apache.org/components/artemis/documentation/javadocs/javadoc-latest/org/apache/activemq/artemis/api/core/client/SendAcknowledgementHandler.html - configuring confirmation window is required for async message sending with Artemis * [fix][artemis] Fix reporting of failed sends * [fix][artemis] Use async sending and acknowledgements * Reformat with spotless --- .../driver/artemis/ArtemisBenchmarkDriver.java | 9 ++++++++- .../driver/artemis/ArtemisBenchmarkProducer.java | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java index f1a966ca2..67db26990 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkDriver.java @@ -49,7 +49,14 @@ public void initialize(File configurationFile, StatsLogger statsLogger) throws I log.info("ActiveMQ Artemis driver configuration: {}", writer.writeValueAsString(config)); try { ServerLocator serverLocator = ActiveMQClient.createServerLocator(config.brokerAddress); - serverLocator.setConfirmationWindowSize(1000); + // confirmation window size is in bytes, set to 1MB + serverLocator.setConfirmationWindowSize(1024 * 1024); + // use asynchronous sending of messages where SendAcknowledgementHandler reports + // success/failure + serverLocator.setBlockOnDurableSend(false); + serverLocator.setBlockOnNonDurableSend(false); + // use async acknowledgement + serverLocator.setBlockOnAcknowledge(false); sessionFactory = serverLocator.createSessionFactory(); session = sessionFactory.createSession(); diff --git a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java index 622d37ea8..0b40546fa 100644 --- a/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java +++ b/driver-artemis/src/main/java/io/openmessaging/benchmark/driver/artemis/ArtemisBenchmarkProducer.java @@ -18,10 +18,12 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.activemq.artemis.api.core.ActiveMQException; +import org.apache.activemq.artemis.api.core.Message; import org.apache.activemq.artemis.api.core.client.ClientMessage; import org.apache.activemq.artemis.api.core.client.ClientProducer; import org.apache.activemq.artemis.api.core.client.ClientSession; import org.apache.activemq.artemis.api.core.client.ClientSessionFactory; +import org.apache.activemq.artemis.api.core.client.SendAcknowledgementHandler; public class ArtemisBenchmarkProducer implements BenchmarkProducer { @@ -51,8 +53,16 @@ public CompletableFuture sendAsync(Optional key, byte[] payload) { try { producer.send( msg, - message -> { - future.complete(null); + new SendAcknowledgementHandler() { + @Override + public void sendAcknowledged(Message message) { + future.complete(null); + } + + @Override + public void sendFailed(Message message, Exception e) { + future.completeExceptionally(e); + } }); } catch (ActiveMQException e) { future.completeExceptionally(e); From 910bd7ca2be9a25e9b03b7226668ea018b40414e Mon Sep 17 00:00:00 2001 From: Elliot West Date: Mon, 24 Oct 2022 17:58:12 +0100 Subject: [PATCH 076/103] Use unique names (#344) --- .../deploy/provision-rabbitmq-aws.tf | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf index 9588610f4..49fa67342 100644 --- a/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf +++ b/driver-rabbitmq/deploy/provision-rabbitmq-aws.tf @@ -1,3 +1,14 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + random = { + source = "hashicorp/random" + } + } +} + variable "public_key_path" { description = < Date: Mon, 24 Oct 2022 17:58:22 +0100 Subject: [PATCH 077/103] Use unique names (#345) # Conflicts: # driver-nats/deploy/provision-nats-aws.tf --- driver-nats/deploy/provision-nats-aws.tf | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/driver-nats/deploy/provision-nats-aws.tf b/driver-nats/deploy/provision-nats-aws.tf index 76d51c09f..3307db02f 100644 --- a/driver-nats/deploy/provision-nats-aws.tf +++ b/driver-nats/deploy/provision-nats-aws.tf @@ -1,3 +1,14 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + random = { + source = "hashicorp/random" + } + } +} + variable "public_key_path" { description = < Date: Mon, 24 Oct 2022 17:58:38 +0100 Subject: [PATCH 078/103] New max sustainable rate implementation (#329) ## Motivation Currently the WorkloadGenerator.findMaximumSustainableRate algorithm introduces publish delay. It has been observed the algorithm settles on a publish rate that neither producers nor consumers can keep up with. For example, on a Kafka workload with 100 topics, 1kb message, the algorithm settled on a publish rate of 2.5m msg/s when the producers could only actually achieve 2m msg/s. ## Changes Implement a new algorithm that checks for both receive backlog and publish backlog and adjusts the publish according and has a progressive rate ramp up when there is no observed backlog. --- benchmark-framework/pom.xml | 22 +++- .../benchmark/RateController.java | 96 +++++++++++++++++ .../benchmark/WorkloadGenerator.java | 101 +----------------- .../io/openmessaging/benchmark/utils/Env.java | 34 ++++++ .../benchmark/RateControllerTest.java | 70 ++++++++++++ .../benchmark/utils/EnvTest.java | 59 ++++++++++ .../src/test/resources/log4j2.yaml | 33 ++++++ driver-rocketmq/pom.xml | 6 ++ pom.xml | 4 + 9 files changed, 325 insertions(+), 100 deletions(-) create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/RateController.java create mode 100644 benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Env.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/RateControllerTest.java create mode 100644 benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/EnvTest.java create mode 100644 benchmark-framework/src/test/resources/log4j2.yaml diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index ea9949af9..256222f9d 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -118,10 +118,6 @@ org.apache.logging.log4j log4j-slf4j-impl - - org.assertj - assertj-core - org.asynchttpclient async-http-client @@ -143,13 +139,31 @@ HdrHistogram 2.1.12 + + org.projectlombok + lombok + provided + + + com.github.stefanbirkner + system-lambda + 1.2.1 + test + + + org.assertj + assertj-core + test + org.junit.jupiter junit-jupiter + test org.mockito mockito-junit-jupiter + test diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/RateController.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/RateController.java new file mode 100644 index 000000000..edb4b9165 --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/RateController.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static lombok.AccessLevel.PACKAGE; + +import io.openmessaging.benchmark.utils.Env; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +class RateController { + private static final long ONE_SECOND_IN_NANOS = SECONDS.toNanos(1); + private final long publishBacklogLimit; + private final long receiveBacklogLimit; + private final double minRampingFactor; + private final double maxRampingFactor; + + @Getter(PACKAGE) + private double rampingFactor; + + private long previousTotalPublished = 0; + private long previousTotalReceived = 0; + + RateController() { + publishBacklogLimit = Env.getLong("PUBLISH_BACKLOG_LIMIT", 1_000); + receiveBacklogLimit = Env.getLong("RECEIVE_BACKLOG_LIMIT", 1_000); + minRampingFactor = Env.getDouble("MIN_RAMPING_FACTOR", 0.01); + maxRampingFactor = Env.getDouble("MAX_RAMPING_FACTOR", 1); + rampingFactor = maxRampingFactor; + } + + double nextRate(double rate, long periodNanos, long totalPublished, long totalReceived) { + long expected = (long) ((rate / ONE_SECOND_IN_NANOS) * periodNanos); + long published = totalPublished - previousTotalPublished; + long received = totalReceived - previousTotalReceived; + + previousTotalPublished = totalPublished; + previousTotalReceived = totalReceived; + + if (log.isDebugEnabled()) { + log.debug( + "Current rate: {} -- Publish rate {} -- Receive Rate: {}", + rate, + rate(published, periodNanos), + rate(received, periodNanos)); + } + + long receiveBacklog = totalPublished - totalReceived; + if (receiveBacklog > receiveBacklogLimit) { + return nextRate(periodNanos, received, expected, receiveBacklog, "Receive"); + } + + long publishBacklog = expected - published; + if (publishBacklog > publishBacklogLimit) { + return nextRate(periodNanos, published, expected, publishBacklog, "Publish"); + } + + rampUp(); + + return rate + (rate * rampingFactor); + } + + private double nextRate(long periodNanos, long actual, long expected, long backlog, String type) { + log.debug("{} backlog: {}", type, backlog); + rampDown(); + long nextExpected = Math.max(0, expected - backlog); + double nextExpectedRate = rate(nextExpected, periodNanos); + double actualRate = rate(actual, periodNanos); + return Math.min(actualRate, nextExpectedRate); + } + + private double rate(long count, long periodNanos) { + return (count / (double) periodNanos) * ONE_SECOND_IN_NANOS; + } + + private void rampUp() { + rampingFactor = Math.min(maxRampingFactor, rampingFactor * 2); + } + + private void rampDown() { + rampingFactor = Math.max(minRampingFactor, rampingFactor / 2); + } +} diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java index 791686d0e..dc91353ee 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/WorkloadGenerator.java @@ -191,20 +191,13 @@ private void ensureTopicsAreReady() throws IOException { * * @param currentRate */ - @SuppressWarnings("checkstyle:LineLength") private void findMaximumSustainableRate(double currentRate) throws IOException { - double maxRate = Double.MAX_VALUE; // Discovered max sustainable rate - double minRate = 0.1; - CountersStats stats = worker.getCountersStats(); - long localTotalMessagesSentCounter = stats.messagesSent; - long localTotalMessagesReceivedCounter = stats.messagesReceived; - int controlPeriodMillis = 3000; long lastControlTimestamp = System.nanoTime(); - int successfulPeriods = 0; + RateController rateController = new RateController(); while (!runCompleted) { // Check every few seconds and adjust the rate @@ -217,97 +210,13 @@ private void findMaximumSustainableRate(double currentRate) throws IOException { // Consider multiple copies when using multiple subscriptions stats = worker.getCountersStats(); long currentTime = System.nanoTime(); - long totalMessagesSent = stats.messagesSent; - long totalMessagesReceived = stats.messagesReceived; - long messagesPublishedInPeriod = totalMessagesSent - localTotalMessagesSentCounter; - long messagesReceivedInPeriod = totalMessagesReceived - localTotalMessagesReceivedCounter; - double publishRateInLastPeriod = - messagesPublishedInPeriod - / (double) (currentTime - lastControlTimestamp) - * TimeUnit.SECONDS.toNanos(1); - double receiveRateInLastPeriod = - messagesReceivedInPeriod - / (double) (currentTime - lastControlTimestamp) - * TimeUnit.SECONDS.toNanos(1); - - if (log.isDebugEnabled()) { - log.debug( - "total-send: {} -- total-received: {} -- int-sent: {} -- int-received: {} -- sent-rate: {} -- received-rate: {}", - totalMessagesSent, - totalMessagesReceived, - messagesPublishedInPeriod, - messagesReceivedInPeriod, - publishRateInLastPeriod, - receiveRateInLastPeriod); - } + long periodNanos = currentTime - lastControlTimestamp; - localTotalMessagesSentCounter = totalMessagesSent; - localTotalMessagesReceivedCounter = totalMessagesReceived; lastControlTimestamp = currentTime; - if (log.isDebugEnabled()) { - log.debug( - "Current rate: {} -- Publish rate {} -- Consume Rate: {} -- min-rate: {} -- max-rate: {}", - dec.format(currentRate), - dec.format(publishRateInLastPeriod), - dec.format(receiveRateInLastPeriod), - dec.format(minRate), - dec.format(maxRate)); - } - - if (publishRateInLastPeriod < currentRate * 0.95) { - // Producer is not able to publish as fast as requested - maxRate = currentRate * 1.1; - currentRate = minRate + (currentRate - minRate) / 2; - - log.debug("Publishers are not meeting requested rate. reducing to {}", currentRate); - } else if (receiveRateInLastPeriod < publishRateInLastPeriod * 0.98) { - // If the consumers are building backlog, we should slow down publish rate - maxRate = currentRate; - currentRate = minRate + (currentRate - minRate) / 2; - log.debug("Consumers are not meeting requested rate. reducing to {}", currentRate); - - // Slows the publishes to let the consumer time to absorb the backlog - worker.adjustPublishRate(minRate / 10); - while (true) { - stats = worker.getCountersStats(); - long backlog = - workload.subscriptionsPerTopic * stats.messagesSent - stats.messagesReceived; - if (backlog < 1000) { - break; - } - - try { - Thread.sleep(100); - } catch (InterruptedException e) { - return; - } - } - - log.debug("Resuming load at reduced rate"); - worker.adjustPublishRate(currentRate); - - try { - // Wait some more time for the publish rate to catch up - Thread.sleep(500); - } catch (InterruptedException e) { - return; - } - - stats = worker.getCountersStats(); - localTotalMessagesSentCounter = stats.messagesSent; - localTotalMessagesReceivedCounter = stats.messagesReceived; - - } else if (currentRate < maxRate) { - minRate = currentRate; - currentRate = Math.min(currentRate * 2, maxRate); - log.debug("No bottleneck found, increasing the rate to {}", currentRate); - } else if (++successfulPeriods > 3) { - minRate = currentRate * 0.95; - maxRate = currentRate * 1.05; - successfulPeriods = 0; - } - + currentRate = + rateController.nextRate( + currentRate, periodNanos, stats.messagesSent, stats.messagesReceived); worker.adjustPublishRate(currentRate); } } diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Env.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Env.java new file mode 100644 index 000000000..366b2ca12 --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/Env.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.utils; + + +import java.util.Optional; +import java.util.function.Function; + +public final class Env { + private Env() {} + + public static long getLong(String key, long defaultValue) { + return get(key, Long::parseLong, defaultValue); + } + + public static double getDouble(String key, double defaultValue) { + return get(key, Double::parseDouble, defaultValue); + } + + public static T get(String key, Function function, T defaultValue) { + return Optional.ofNullable(System.getenv(key)).map(function).orElse(defaultValue); + } +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/RateControllerTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/RateControllerTest.java new file mode 100644 index 000000000..6f014f019 --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/RateControllerTest.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class RateControllerTest { + private final RateController rateController = new RateController(); + private double rate = 10_000; + private long periodNanos = SECONDS.toNanos(1); + + @Test + void receiveBacklog() { + assertThat(rateController.getRampingFactor()).isEqualTo(1); + + // no backlog + rate = rateController.nextRate(rate, periodNanos, 10_000, 10_000); + assertThat(rate).isEqualTo(20_000); + assertThat(rateController.getRampingFactor()).isEqualTo(1); + + // receive backlog + rate = rateController.nextRate(rate, periodNanos, 20_000, 15_000); + assertThat(rate).isEqualTo(5_000); + assertThat(rateController.getRampingFactor()).isEqualTo(0.5); + } + + @Test + void publishBacklog() { + assertThat(rateController.getRampingFactor()).isEqualTo(1); + + // no backlog + rate = rateController.nextRate(rate, periodNanos, 10_000, 10_000); + assertThat(rate).isEqualTo(20_000); + assertThat(rateController.getRampingFactor()).isEqualTo(1); + + // publish backlog + rate = rateController.nextRate(rate, periodNanos, 15_000, 20_000); + assertThat(rate).isEqualTo(5_000); + assertThat(rateController.getRampingFactor()).isEqualTo(0.5); + } + + @Test + void rampUp() { + assertThat(rateController.getRampingFactor()).isEqualTo(1); + + // receive backlog + rate = rateController.nextRate(rate, periodNanos, 10_000, 5_000); + assertThat(rate).isEqualTo(5_000); + assertThat(rateController.getRampingFactor()).isEqualTo(0.5); + + // no backlog + rate = rateController.nextRate(rate, periodNanos, 20_000, 20_000); + assertThat(rate).isEqualTo(10_000); + assertThat(rateController.getRampingFactor()).isEqualTo(1); + } +} diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/EnvTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/EnvTest.java new file mode 100644 index 000000000..cf62b85dd --- /dev/null +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/EnvTest.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.utils; + +import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable; +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class EnvTest { + private static final String ENV_KEY = "KEY"; + + @Test + void envLong() throws Exception { + withEnvironmentVariable(ENV_KEY, "2") + .execute( + () -> { + assertThat(Env.getLong(ENV_KEY, 1)).isEqualTo(2); + }); + } + + @Test + void envLongDefault() throws Exception { + withEnvironmentVariable(ENV_KEY, null) + .execute( + () -> { + assertThat(Env.getLong(ENV_KEY, 1)).isEqualTo(1); + }); + } + + @Test + void envDouble() throws Exception { + withEnvironmentVariable(ENV_KEY, "2.34") + .execute( + () -> { + assertThat(Env.getDouble(ENV_KEY, 1.23)).isEqualTo(2.34); + }); + } + + @Test + void envDoubleDefault() throws Exception { + withEnvironmentVariable(ENV_KEY, null) + .execute( + () -> { + assertThat(Env.getDouble(ENV_KEY, 1.23)).isEqualTo(1.23); + }); + } +} diff --git a/benchmark-framework/src/test/resources/log4j2.yaml b/benchmark-framework/src/test/resources/log4j2.yaml new file mode 100644 index 000000000..7ed3f311f --- /dev/null +++ b/benchmark-framework/src/test/resources/log4j2.yaml @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Configuration: + status: INFO + name: messaging-benchmark + + Appenders: + Console: + name: Console + target: SYSTEM_OUT + PatternLayout: + Pattern: "%d{HH:mm:ss.SSS} [%t] %-4level %c{1} - %msg%n" + Loggers: + Logger: + - name: io.openmessaging + level: debug + Root: + level: info + additivity: false + AppenderRef: + - ref: Console diff --git a/driver-rocketmq/pom.xml b/driver-rocketmq/pom.xml index a251ada24..9e60ff3cc 100644 --- a/driver-rocketmq/pom.xml +++ b/driver-rocketmq/pom.xml @@ -53,6 +53,12 @@ org.apache.rocketmq rocketmq-tools ${rocketmq.version} + + + ch.qos.logback + logback-classic + + diff --git a/pom.xml b/pom.xml index 64981c973..5542f673b 100644 --- a/pom.xml +++ b/pom.xml @@ -350,6 +350,10 @@ maven-surefire-plugin ${maven.surefire.plugin.version} + + + --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED + From 1b48fa721642239681e0a8706ca82474f8ca6ee7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Oct 2022 10:06:26 -0700 Subject: [PATCH 079/103] Bump commons-text from 1.9 to 1.10.0 in /tool (#346) Bumps commons-text from 1.9 to 1.10.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-text dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tool/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/pom.xml b/tool/pom.xml index 97af84940..4c0391582 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -43,7 +43,7 @@ org.apache.commons commons-text - 1.9 + 1.10.0 org.apache.logging.log4j From 276693b62623f3d415eb400a7f839314004aeac5 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Wed, 26 Oct 2022 03:55:39 +0100 Subject: [PATCH 080/103] Increase buffer sizes (#349) --- .../benchmark/worker/jackson/HistogramDeserializer.java | 8 +++----- .../benchmark/worker/jackson/HistogramSerializer.java | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java index 9817a6301..6aa562bed 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramDeserializer.java @@ -31,7 +31,7 @@ public class HistogramDeserializer extends StdDeserializer { private final ThreadLocal threadBuffer = - ThreadLocal.withInitial(() -> ByteBuffer.allocate(1024 * 1024)); + ThreadLocal.withInitial(() -> ByteBuffer.allocate(8 * 1024 * 1024)); public HistogramDeserializer() { super(Histogram.class); @@ -46,10 +46,8 @@ public Histogram deserialize(JsonParser jsonParser, DeserializationContext deser jsonParser.readBinaryValue(os); buffer.flip(); // Long.MIN_VALUE used so that Histogram will defer to the value encoded in the histogram - // value. This - // assumes that it is acceptable for the deserialized value we create to share the same - // parameters of the - // source histogram that was serialized. + // value. This assumes that it is acceptable for the deserialized value we create to + // share the same parameters of the source histogram that was serialized. return Histogram.decodeFromCompressedByteBuffer(buffer, Long.MIN_VALUE); } catch (Exception e) { log.error( diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java index 55703bf0e..a1b61afa3 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java @@ -24,7 +24,7 @@ public class HistogramSerializer extends StdSerializer { private final ThreadLocal threadBuffer = - ThreadLocal.withInitial(() -> ByteBuffer.allocate(1024 * 1024)); + ThreadLocal.withInitial(() -> ByteBuffer.allocate(8 * 1024 * 1024)); public HistogramSerializer() { super(Histogram.class); From 1c3eece43d40a225b82b2ef9485730bc74c4e6db Mon Sep 17 00:00:00 2001 From: Elliot West Date: Wed, 26 Oct 2022 03:56:17 +0100 Subject: [PATCH 081/103] NATS config changes (#348) --- driver-nats/deploy/templates/cluster.conf | 1 + driver-nats/nats.yaml | 2 +- .../benchmark/driver/nats/NatsBenchmarkDriver.java | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/driver-nats/deploy/templates/cluster.conf b/driver-nats/deploy/templates/cluster.conf index 48f998447..1b1c06250 100644 --- a/driver-nats/deploy/templates/cluster.conf +++ b/driver-nats/deploy/templates/cluster.conf @@ -18,6 +18,7 @@ server_name: {{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] http: {{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}:8222 jetstream: enabled +write_deadline: 60 jetstream { store_dir: /mnt/data diff --git a/driver-nats/nats.yaml b/driver-nats/nats.yaml index f5bbde94c..8a51250e6 100644 --- a/driver-nats/nats.yaml +++ b/driver-nats/nats.yaml @@ -17,4 +17,4 @@ driverClass: io.openmessaging.benchmark.driver.nats.NatsBenchmarkDriver natsHostUrl: localhost:4222 -replicationFactor: 1 +replicationFactor: 3 diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index aa2989254..866e8ea43 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -37,6 +37,7 @@ import io.openmessaging.benchmark.driver.ConsumerCallback; import java.io.File; import java.io.IOException; +import java.time.Duration; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; @@ -58,6 +59,9 @@ public void initialize(File configurationFile, StatsLogger statsLogger) new Options.Builder() .server(config.natsHostUrl) .maxReconnects(5) + .reconnectWait(Duration.ofSeconds(1)) + .connectionTimeout(Duration.ofSeconds(5)) + .pingInterval(Duration.ofSeconds(60)) .errorListener( new ErrorListener() { @Override From d707b876be7658897b5a5d6494b7f59327d833d8 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Wed, 26 Oct 2022 03:56:53 +0100 Subject: [PATCH 082/103] Allow deployment of benchmark code in isolation (#347) From dc4da71e78fc99263911f2a9c70e069221591cd9 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Thu, 27 Oct 2022 16:46:32 +0100 Subject: [PATCH 083/103] [Feature] Support shared subscriptions (#353) * Configurable pulsar subscription type * End with newline * Reformat pulsar driver * Support shared subscriptions --- .../benchmark/driver/nats/NatsBenchmarkDriver.java | 3 ++- driver-pulsar/pulsar.yaml | 3 ++- .../benchmark/driver/pulsar/PulsarBenchmarkDriver.java | 3 +-- .../benchmark/driver/pulsar/config/PulsarConsumerConfig.java | 4 ++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index 866e8ea43..93882ff79 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; public class NatsBenchmarkDriver implements BenchmarkDriver { + private NatsConfig config; private Connection connection; @@ -126,7 +127,7 @@ public CompletableFuture createConsumer( msg.ack(); }, false, - new PushSubscribeOptions.Builder().build()); + new PushSubscribeOptions.Builder().deliverGroup(subscriptionName).build()); return CompletableFuture.completedFuture(new NatsBenchmarkConsumer()); } catch (Exception e) { CompletableFuture f = new CompletableFuture<>(); diff --git a/driver-pulsar/pulsar.yaml b/driver-pulsar/pulsar.yaml index 6a10e8055..bbb46a996 100644 --- a/driver-pulsar/pulsar.yaml +++ b/driver-pulsar/pulsar.yaml @@ -45,4 +45,5 @@ producer: pendingQueueSize: 0 consumer: - receiverQueueSize: 10000 \ No newline at end of file + receiverQueueSize: 10000 + subscriptionType: Failover diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java index dd02fb874..f65fe4f48 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/PulsarBenchmarkDriver.java @@ -47,7 +47,6 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SizeUnit; -import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.BacklogQuota.RetentionPolicy; import org.apache.pulsar.common.policies.data.PersistencePolicies; @@ -222,7 +221,7 @@ CompletableFuture> createInternalConsumer( return client .newConsumer(Schema.BYTEBUFFER) .priorityLevel(0) - .subscriptionType(SubscriptionType.Failover) + .subscriptionType(config.consumer.subscriptionType) .messageListener( (c, msg) -> { try { diff --git a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java index 981271eec..d3ac54902 100644 --- a/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java +++ b/driver-pulsar/src/main/java/io/openmessaging/benchmark/driver/pulsar/config/PulsarConsumerConfig.java @@ -13,7 +13,11 @@ */ package io.openmessaging.benchmark.driver.pulsar.config; + +import org.apache.pulsar.client.api.SubscriptionType; + public class PulsarConsumerConfig { public int receiverQueueSize = 10000; + public SubscriptionType subscriptionType = SubscriptionType.Failover; } From 620419d20a79049a423bda9fc2be52cbf6d2b8d5 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Mon, 9 Jan 2023 18:28:54 +0000 Subject: [PATCH 084/103] Convenience ansible (#350) Restart Rename for consistency --- driver-nats/deploy/deploy-client-jars.yaml | 30 +++++++++++++++++++ driver-nats/deploy/restart-brokers.yaml | 23 ++++++++++++++ driver-nats/deploy/restart-workers.yaml | 24 +++++++++++++++ driver-pulsar/README.md | 2 +- .../deploy/ssd/deploy-client-jars.yaml | 30 +++++++++++++++++++ ...start_broker.yaml => restart-brokers.yaml} | 0 driver-pulsar/deploy/ssd/restart-workers.yaml | 24 +++++++++++++++ .../deploy/deploy-client-jars.yaml | 30 +++++++++++++++++++ driver-rabbitmq/deploy/restart-brokers.yaml | 23 ++++++++++++++ driver-rabbitmq/deploy/restart-workers.yaml | 24 +++++++++++++++ 10 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 driver-nats/deploy/deploy-client-jars.yaml create mode 100644 driver-nats/deploy/restart-brokers.yaml create mode 100644 driver-nats/deploy/restart-workers.yaml create mode 100644 driver-pulsar/deploy/ssd/deploy-client-jars.yaml rename driver-pulsar/deploy/ssd/{restart_broker.yaml => restart-brokers.yaml} (100%) create mode 100644 driver-pulsar/deploy/ssd/restart-workers.yaml create mode 100644 driver-rabbitmq/deploy/deploy-client-jars.yaml create mode 100644 driver-rabbitmq/deploy/restart-brokers.yaml create mode 100644 driver-rabbitmq/deploy/restart-workers.yaml diff --git a/driver-nats/deploy/deploy-client-jars.yaml b/driver-nats/deploy/deploy-client-jars.yaml new file mode 100644 index 000000000..d210cba01 --- /dev/null +++ b/driver-nats/deploy/deploy-client-jars.yaml @@ -0,0 +1,30 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- name: Deploy benchmark + hosts: client + connection: ssh + become: true + tasks: + - shell: rm -fr /opt/openmessaging-benchmark-0.0.1-SNAPSHOT + - shell: rm -f /opt/benchmark/lib/io.openmessaging.* + - unarchive: + src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz + dest: /opt + - shell: cp -f /opt/openmessaging-benchmark-0.0.1-SNAPSHOT/lib/io.openmessaging.* /opt/benchmark/lib/ + - name: Benchmark - Start service + systemd: + state: restarted + daemon_reload: yes + name: "benchmark-worker" diff --git a/driver-nats/deploy/restart-brokers.yaml b/driver-nats/deploy/restart-brokers.yaml new file mode 100644 index 000000000..fd14cf88c --- /dev/null +++ b/driver-nats/deploy/restart-brokers.yaml @@ -0,0 +1,23 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- name: Restart brokers + hosts: nats + connection: ssh + become: true + tasks: + - name: NATS - Start service + systemd: + state: restarted + name: "nats-server" diff --git a/driver-nats/deploy/restart-workers.yaml b/driver-nats/deploy/restart-workers.yaml new file mode 100644 index 000000000..2c1e563b7 --- /dev/null +++ b/driver-nats/deploy/restart-workers.yaml @@ -0,0 +1,24 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- name: Benchmarking worker restart + hosts: client + connection: ssh + become: true + tasks: + - name: Benchmark - Start service + systemd: + state: restarted + daemon_reload: yes + name: "benchmark-worker" diff --git a/driver-pulsar/README.md b/driver-pulsar/README.md index 53ed1cb63..730b6f562 100644 --- a/driver-pulsar/README.md +++ b/driver-pulsar/README.md @@ -60,6 +60,6 @@ TF_STATE=. ansible-playbook \ --user ec2-user \ --inventory `which terraform-inventory` \ -e @extra_vars.yaml \ - restart_broker.yaml + restart-brokers.yaml ``` diff --git a/driver-pulsar/deploy/ssd/deploy-client-jars.yaml b/driver-pulsar/deploy/ssd/deploy-client-jars.yaml new file mode 100644 index 000000000..d210cba01 --- /dev/null +++ b/driver-pulsar/deploy/ssd/deploy-client-jars.yaml @@ -0,0 +1,30 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- name: Deploy benchmark + hosts: client + connection: ssh + become: true + tasks: + - shell: rm -fr /opt/openmessaging-benchmark-0.0.1-SNAPSHOT + - shell: rm -f /opt/benchmark/lib/io.openmessaging.* + - unarchive: + src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz + dest: /opt + - shell: cp -f /opt/openmessaging-benchmark-0.0.1-SNAPSHOT/lib/io.openmessaging.* /opt/benchmark/lib/ + - name: Benchmark - Start service + systemd: + state: restarted + daemon_reload: yes + name: "benchmark-worker" diff --git a/driver-pulsar/deploy/ssd/restart_broker.yaml b/driver-pulsar/deploy/ssd/restart-brokers.yaml similarity index 100% rename from driver-pulsar/deploy/ssd/restart_broker.yaml rename to driver-pulsar/deploy/ssd/restart-brokers.yaml diff --git a/driver-pulsar/deploy/ssd/restart-workers.yaml b/driver-pulsar/deploy/ssd/restart-workers.yaml new file mode 100644 index 000000000..2c1e563b7 --- /dev/null +++ b/driver-pulsar/deploy/ssd/restart-workers.yaml @@ -0,0 +1,24 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- name: Benchmarking worker restart + hosts: client + connection: ssh + become: true + tasks: + - name: Benchmark - Start service + systemd: + state: restarted + daemon_reload: yes + name: "benchmark-worker" diff --git a/driver-rabbitmq/deploy/deploy-client-jars.yaml b/driver-rabbitmq/deploy/deploy-client-jars.yaml new file mode 100644 index 000000000..d210cba01 --- /dev/null +++ b/driver-rabbitmq/deploy/deploy-client-jars.yaml @@ -0,0 +1,30 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- name: Deploy benchmark + hosts: client + connection: ssh + become: true + tasks: + - shell: rm -fr /opt/openmessaging-benchmark-0.0.1-SNAPSHOT + - shell: rm -f /opt/benchmark/lib/io.openmessaging.* + - unarchive: + src: ../../package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz + dest: /opt + - shell: cp -f /opt/openmessaging-benchmark-0.0.1-SNAPSHOT/lib/io.openmessaging.* /opt/benchmark/lib/ + - name: Benchmark - Start service + systemd: + state: restarted + daemon_reload: yes + name: "benchmark-worker" diff --git a/driver-rabbitmq/deploy/restart-brokers.yaml b/driver-rabbitmq/deploy/restart-brokers.yaml new file mode 100644 index 000000000..35d269238 --- /dev/null +++ b/driver-rabbitmq/deploy/restart-brokers.yaml @@ -0,0 +1,23 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- name: Restart brokers + hosts: rabbitmq + connection: ssh + become: true + tasks: + - name: RabbitMQ - Start service + systemd: + state: restarted + name: "rabbitmq-server" diff --git a/driver-rabbitmq/deploy/restart-workers.yaml b/driver-rabbitmq/deploy/restart-workers.yaml new file mode 100644 index 000000000..2c1e563b7 --- /dev/null +++ b/driver-rabbitmq/deploy/restart-workers.yaml @@ -0,0 +1,24 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- name: Benchmarking worker restart + hosts: client + connection: ssh + become: true + tasks: + - name: Benchmark - Start service + systemd: + state: restarted + daemon_reload: yes + name: "benchmark-worker" From 1066759a0d346722087a891a940ffc6ee63044b5 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Mon, 9 Jan 2023 18:31:36 +0000 Subject: [PATCH 085/103] Create and share consumer (#356) --- driver-nats/deploy/deploy.yaml | 2 +- driver-nats/pom.xml | 2 +- .../driver/nats/NatsBenchmarkDriver.java | 19 +++++++++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/driver-nats/deploy/deploy.yaml b/driver-nats/deploy/deploy.yaml index 3ab205213..40716ab23 100644 --- a/driver-nats/deploy/deploy.yaml +++ b/driver-nats/deploy/deploy.yaml @@ -59,7 +59,7 @@ - name: NATS - Install server tags: [ broker ] yum: - name: "https://github.com/nats-io/nats-server/releases/download/v2.9.3/nats-server-v2.9.3-amd64.rpm" + name: "https://github.com/nats-io/nats-server/releases/download/v2.9.6/nats-server-v2.9.6-amd64.rpm" state: present disable_gpg_check: yes - name: NATS - Apply server configuration diff --git a/driver-nats/pom.xml b/driver-nats/pom.xml index d2d6bc6d1..0d2052650 100644 --- a/driver-nats/pom.xml +++ b/driver-nats/pom.xml @@ -33,7 +33,7 @@ io.nats jnats - 2.15.6 + 2.16.3 diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index 93882ff79..c5b677f30 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -27,6 +27,7 @@ import io.nats.client.Nats; import io.nats.client.Options; import io.nats.client.PushSubscribeOptions; +import io.nats.client.api.ConsumerConfiguration; import io.nats.client.api.StorageType; import io.nats.client.api.StreamConfiguration; import io.nats.client.api.StreamInfo; @@ -44,11 +45,10 @@ import org.slf4j.LoggerFactory; public class NatsBenchmarkDriver implements BenchmarkDriver { - private NatsConfig config; - private Connection connection; private JetStream jetStream; + private JetStreamManagement jetStreamManagement; @Override public void initialize(File configurationFile, StatsLogger statsLogger) @@ -77,6 +77,7 @@ public void exceptionOccurred(Connection conn, Exception exp) { }) .build()); this.jetStream = connection.jetStream(); + this.jetStreamManagement = connection.jetStreamManagement(); } @Override @@ -114,9 +115,19 @@ public CompletableFuture createProducer(String topic) { public CompletableFuture createConsumer( String topic, String subscriptionName, ConsumerCallback consumerCallback) { - Dispatcher dispatcher = connection.createDispatcher(); + ConsumerConfiguration cc = + ConsumerConfiguration.builder() + .durable("durable-" + subscriptionName) + .deliverSubject("delivery-subject-" + subscriptionName) + .deliverGroup("group-" + subscriptionName) + .build(); + PushSubscribeOptions pso = PushSubscribeOptions.builder().configuration(cc).build(); try { + jetStreamManagement.addOrUpdateConsumer(topic, cc); + + Dispatcher dispatcher = connection.createDispatcher(); + JetStreamSubscription sub = jetStream.subscribe( topic, @@ -127,7 +138,7 @@ public CompletableFuture createConsumer( msg.ack(); }, false, - new PushSubscribeOptions.Builder().deliverGroup(subscriptionName).build()); + pso); return CompletableFuture.completedFuture(new NatsBenchmarkConsumer()); } catch (Exception e) { CompletableFuture f = new CompletableFuture<>(); From 7f910c40bc03c59ce9cb8df94350a8e21f4b522a Mon Sep 17 00:00:00 2001 From: Elliot West Date: Mon, 9 Jan 2023 18:32:38 +0000 Subject: [PATCH 086/103] Stream type (#357) --- .../benchmark/driver/rabbitmq/RabbitMqConfig.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java index 24bec1876..40bd1cbb3 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java @@ -38,6 +38,12 @@ Map queueOptions() { Map queueOptions() { return Collections.singletonMap("x-queue-type", "quorum"); } + }, + STREAM { + @Override + Map queueOptions() { + return Collections.singletonMap("x-queue-type", "stream"); + } }; abstract Map queueOptions(); From 9cb676414050bc2722e331f429ad50c1f99c6ddf Mon Sep 17 00:00:00 2001 From: Elliot West Date: Mon, 9 Jan 2023 18:33:18 +0000 Subject: [PATCH 087/103] Set partition handling behaviour (#358) License --- driver-rabbitmq/deploy/deploy.yaml | 5 +++++ driver-rabbitmq/deploy/templates/rabbitmq.conf | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 driver-rabbitmq/deploy/templates/rabbitmq.conf diff --git a/driver-rabbitmq/deploy/deploy.yaml b/driver-rabbitmq/deploy/deploy.yaml index 29682afa1..cc5909b7d 100644 --- a/driver-rabbitmq/deploy/deploy.yaml +++ b/driver-rabbitmq/deploy/deploy.yaml @@ -64,6 +64,11 @@ state: present disable_gpg_check: yes + - name: RabbitMQ - Create rabbitmq.conf file + template: + src: "templates/rabbitmq.conf" + dest: "/etc/rabbitmq/rabbitmq.conf" + - name: RabbitMQ - Create rabbitmq-env.conf file template: src: "templates/rabbitmq-env.conf" diff --git a/driver-rabbitmq/deploy/templates/rabbitmq.conf b/driver-rabbitmq/deploy/templates/rabbitmq.conf new file mode 100644 index 000000000..c8541df14 --- /dev/null +++ b/driver-rabbitmq/deploy/templates/rabbitmq.conf @@ -0,0 +1,17 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## Pauses all nodes on the minority side of a partition. The cluster +## MUST have an odd number of nodes (3, 5, etc) +cluster_partition_handling = pause_minority From d04724a469a71c544c2e67beddc5d53f4095f2e9 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Mon, 9 Jan 2023 18:33:29 +0000 Subject: [PATCH 088/103] Clients require all broker address to fail over (#359) Update comment Spotless Pass credentials Remove final Typo --- .../rabbitmq/RabbitMqBenchmarkDriver.java | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index 2ebc4bdcc..81ed88c35 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -13,10 +13,12 @@ */ package io.openmessaging.benchmark.driver.rabbitmq; +import static java.util.stream.Collectors.toList; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.rabbitmq.client.Address; import com.rabbitmq.client.AlreadyClosedException; import com.rabbitmq.client.BuiltinExchangeType; import com.rabbitmq.client.Channel; @@ -30,6 +32,7 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -37,6 +40,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; import org.apache.bookkeeper.stats.StatsLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +49,10 @@ public class RabbitMqBenchmarkDriver implements BenchmarkDriver { private RabbitMqConfig config; private final AtomicInteger uriIndex = new AtomicInteger(); + /** + * Map of client's primary broker to the connection -- the connection may still be able to fall + * back to secondary brokers. + */ private final Map connections = new ConcurrentHashMap<>(); @Override @@ -70,9 +78,10 @@ public void close() { @Override public String getTopicNamePrefix() { - // Do a round-robin on AMQP URIs - URI configUri = - URI.create(config.amqpUris.get(uriIndex.getAndIncrement() % config.amqpUris.size())); + // Distribute topics by performing a round-robin on AMQP URIs + String primaryBrokerUri = + config.amqpUris.get(uriIndex.getAndIncrement() % config.amqpUris.size()); + URI configUri = URI.create(primaryBrokerUri); URI topicUri = configUri.resolve(configUri.getRawPath() + "?exchange=test-exchange"); return topicUri.toString(); } @@ -150,15 +159,26 @@ private String getExchangeName(String uri) { return parameters.get("exchange").get(0); } - private Connection getOrCreateConnection(String uri) { + private Connection getOrCreateConnection(String primaryBrokerUri) { return connections.computeIfAbsent( - uri, - uriKey -> { + primaryBrokerUri, + p -> { + String[] userInfo = newURI(primaryBrokerUri).getUserInfo().split(":"); + String user = userInfo[0]; + String password = userInfo[1]; + // RabbitMQ will pick the first available address from the list. Future reconnection + // attempts will pick a random accessible address from the provided list. + List
addresses = + Stream.concat(Stream.of(p), config.amqpUris.stream().filter(s -> !s.equals(p))) + .map(s -> newURI(s)) + .map(u -> new Address(u.getHost(), u.getPort())) + .collect(toList()); try { ConnectionFactory connectionFactory = new ConnectionFactory(); connectionFactory.setAutomaticRecoveryEnabled(true); - connectionFactory.setUri(uri); - return connectionFactory.newConnection(); + connectionFactory.setUsername(user); + connectionFactory.setPassword(password); + return connectionFactory.newConnection(addresses); } catch (Exception e) { throw new RuntimeException("Couldn't establish connection", e); } @@ -170,4 +190,12 @@ private Connection getOrCreateConnection(String uri) { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private static final Logger log = LoggerFactory.getLogger(RabbitMqBenchmarkDriver.class); + + private static URI newURI(String uri) { + try { + return new URI(uri); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } } From c3e271d0d7c1845cd9fa0a9bb7067504f0dd1912 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Wed, 25 Jan 2023 21:30:36 +0000 Subject: [PATCH 089/103] Resilient resource creation (#362) * Resilient resource creation # Conflicts: # benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java # driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java Resilient resource creation # Conflicts: # benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java # driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java # driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml License header Missing import spotless * findbugs exclude --- .../benchmark/worker/LocalWorker.java | 33 +++++-- .../benchmark/driver/BenchmarkDriver.java | 46 +++++++++ .../benchmark/driver/ResourceCreator.java | 99 +++++++++++++++++++ .../deploy/templates/rabbitmq-quorum.yaml | 9 +- driver-rabbitmq/rabbitmq.yaml | 7 +- .../rabbitmq/RabbitMqBenchmarkDriver.java | 68 +++++++++++-- .../driver/rabbitmq/RabbitMqConfig.java | 4 + etc/findbugsExclude.xml | 4 + 8 files changed, 252 insertions(+), 18 deletions(-) create mode 100644 driver-api/src/main/java/io/openmessaging/benchmark/driver/ResourceCreator.java diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java index b6416427e..b968b5ce7 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/LocalWorker.java @@ -24,6 +24,8 @@ import io.openmessaging.benchmark.DriverConfiguration; import io.openmessaging.benchmark.driver.BenchmarkConsumer; import io.openmessaging.benchmark.driver.BenchmarkDriver; +import io.openmessaging.benchmark.driver.BenchmarkDriver.ConsumerInfo; +import io.openmessaging.benchmark.driver.BenchmarkDriver.ProducerInfo; import io.openmessaging.benchmark.driver.BenchmarkDriver.TopicInfo; import io.openmessaging.benchmark.driver.BenchmarkProducer; import io.openmessaging.benchmark.driver.ConsumerCallback; @@ -45,11 +47,11 @@ import java.util.Map; import java.util.Optional; import java.util.TreeMap; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; @@ -125,24 +127,35 @@ private String generateTopicName(int i) { @Override public void createProducers(List topics) { Timer timer = new Timer(); + AtomicInteger index = new AtomicInteger(); - List> futures = - topics.stream().map(topic -> benchmarkDriver.createProducer(topic)).collect(toList()); + producers.addAll( + benchmarkDriver + .createProducers( + topics.stream() + .map(t -> new ProducerInfo(index.getAndIncrement(), t)) + .collect(toList())) + .join()); - futures.forEach(f -> producers.add(f.join())); log.info("Created {} producers in {} ms", producers.size(), timer.elapsedMillis()); } @Override public void createConsumers(ConsumerAssignment consumerAssignment) { Timer timer = new Timer(); + AtomicInteger index = new AtomicInteger(); + + consumers.addAll( + benchmarkDriver + .createConsumers( + consumerAssignment.topicsSubscriptions.stream() + .map( + c -> + new ConsumerInfo( + index.getAndIncrement(), c.topic, c.subscription, this)) + .collect(toList())) + .join()); - List> futures = - consumerAssignment.topicsSubscriptions.stream() - .map(ts -> benchmarkDriver.createConsumer(ts.topic, ts.subscription, this)) - .collect(toList()); - - futures.forEach(f -> consumers.add(f.join())); log.info("Created {} consumers in {} ms", consumers.size(), timer.elapsedMillis()); } diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java index 32f18f082..88c5814aa 100644 --- a/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/BenchmarkDriver.java @@ -13,6 +13,7 @@ */ package io.openmessaging.benchmark.driver; +import static java.util.stream.Collectors.toList; import java.io.File; import java.io.IOException; @@ -75,6 +76,19 @@ default CompletableFuture createTopics(List topicInfos) { */ CompletableFuture createProducer(String topic); + /** + * Create a producers for a given topic. + * + * @param producers + * @return a producers future + */ + default CompletableFuture> createProducers(List producers) { + List> futures = + producers.stream().map(ci -> createProducer(ci.getTopic())).collect(toList()); + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply(v -> futures.stream().map(CompletableFuture::join).collect(toList())); + } + /** * Create a benchmark consumer relative to one particular topic and subscription. * @@ -89,9 +103,41 @@ default CompletableFuture createTopics(List topicInfos) { CompletableFuture createConsumer( String topic, String subscriptionName, ConsumerCallback consumerCallback); + /** + * Create a consumers for a given topic. + * + * @param consumers + * @return a consumers future + */ + default CompletableFuture> createConsumers(List consumers) { + List> futures = + consumers.stream() + .map( + ci -> + createConsumer( + ci.getTopic(), ci.getSubscriptionName(), ci.getConsumerCallback())) + .collect(toList()); + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply(v -> futures.stream().map(CompletableFuture::join).collect(toList())); + } + @Value class TopicInfo { String topic; int partitions; } + + @Value + class ProducerInfo { + int id; + String topic; + } + + @Value + class ConsumerInfo { + int id; + String topic; + String subscriptionName; + ConsumerCallback consumerCallback; + } } diff --git a/driver-api/src/main/java/io/openmessaging/benchmark/driver/ResourceCreator.java b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ResourceCreator.java new file mode 100644 index 000000000..3bdfe5372 --- /dev/null +++ b/driver-api/src/main/java/io/openmessaging/benchmark/driver/ResourceCreator.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.benchmark.driver; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toMap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class ResourceCreator { + private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + private final String name; + private final int maxBatchSize; + private final long interBatchDelayMs; + private final Function, Map>> invokeBatchFn; + private final Function, CreationResult> complete; + + public CompletableFuture> create(List resources) { + return CompletableFuture.completedFuture(createBlocking(resources)); + } + + private List createBlocking(List resources) { + BlockingQueue queue = new ArrayBlockingQueue<>(resources.size(), true, resources); + List batch = new ArrayList<>(); + List created = new ArrayList<>(); + AtomicInteger succeeded = new AtomicInteger(); + + ScheduledFuture loggingFuture = + executor.scheduleAtFixedRate( + () -> log.info("Created {}s {}/{}", name, succeeded.get(), resources.size()), + 10, + 10, + SECONDS); + + try { + while (succeeded.get() < resources.size()) { + int batchSize = queue.drainTo(batch, maxBatchSize); + if (batchSize > 0) { + executeBatch(batch) + .forEach( + (resource, result) -> { + if (result.success) { + created.add(result.created); + succeeded.incrementAndGet(); + } else { + //noinspection ResultOfMethodCallIgnored + queue.offer(resource); + } + }); + batch.clear(); + } + } + } finally { + loggingFuture.cancel(true); + } + return created; + } + + @SneakyThrows + private Map> executeBatch(List batch) { + log.debug("Executing batch, size: {}", batch.size()); + Thread.sleep(interBatchDelayMs); + return invokeBatchFn.apply(batch).entrySet().stream() + .collect(toMap(Map.Entry::getKey, e -> complete.apply(e.getValue()))); + } + + @Value + public static class CreationResult { + C created; + boolean success; + } +} diff --git a/driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml b/driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml index 90711302b..121bfcc0d 100644 --- a/driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml +++ b/driver-rabbitmq/deploy/templates/rabbitmq-quorum.yaml @@ -18,9 +18,16 @@ driverClass: io.openmessaging.benchmark.driver.rabbitmq.RabbitMqBenchmarkDriver # RabbitMq client specific configurations +# RMQ struggles to create more than 10 quroum queues at a time, so we batch with a delay +producerCreationDelay: 100 +producerCreationBatchSize: 5 +consumerCreationDelay: 100 +consumerCreationBatchSize: 5 + amqpUris: {% for pulsar in groups['rabbitmq'] %} - amqp://admin:admin@{{ hostvars[pulsar].private_ip }}:5672 {% endfor %} -messagePersistence: false + +# messagePersistence setting is ignored in the quorum implementation queueType: QUORUM diff --git a/driver-rabbitmq/rabbitmq.yaml b/driver-rabbitmq/rabbitmq.yaml index 0269a8b30..1132a559c 100644 --- a/driver-rabbitmq/rabbitmq.yaml +++ b/driver-rabbitmq/rabbitmq.yaml @@ -18,7 +18,12 @@ driverClass: io.openmessaging.benchmark.driver.rabbitmq.RabbitMqBenchmarkDriver # RabbitMq client specific configurations +producerCreationDelay: 100 +producerCreationBatchSize: 5 +consumerCreationDelay: 100 +consumerCreationBatchSize: 5 + amqpUris: - amqp://localhost messagePersistence: false -queueType: CLASSIC +queueType: QUORUM diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java index 81ed88c35..5d639b545 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqBenchmarkDriver.java @@ -14,6 +14,7 @@ package io.openmessaging.benchmark.driver.rabbitmq; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -29,6 +30,8 @@ import io.openmessaging.benchmark.driver.BenchmarkDriver; import io.openmessaging.benchmark.driver.BenchmarkProducer; import io.openmessaging.benchmark.driver.ConsumerCallback; +import io.openmessaging.benchmark.driver.ResourceCreator; +import io.openmessaging.benchmark.driver.ResourceCreator.CreationResult; import java.io.File; import java.io.IOException; import java.net.URI; @@ -38,6 +41,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; @@ -120,6 +124,57 @@ public CompletableFuture createProducer(String topic) { return future; } + @Override + public CompletableFuture> createProducers(List producers) { + return new ResourceCreator( + "producer", + config.producerCreationBatchSize, + config.producerCreationDelay, + ps -> ps.stream().collect(toMap(p -> p, p -> createProducer(p.getTopic()))), + fc -> { + try { + return new CreationResult<>(fc.get(), true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (ExecutionException e) { + log.debug(e.getMessage()); + return new CreationResult<>(null, false); + } + }) + .create(producers); + } + + @Override + public CompletableFuture> createConsumers(List consumers) { + return new ResourceCreator( + "consumer", + config.consumerCreationBatchSize, + config.consumerCreationDelay, + cs -> + cs.stream() + .collect( + toMap( + c -> c, + c -> + createConsumer( + c.getTopic(), + c.getSubscriptionName(), + c.getConsumerCallback()))), + fc -> { + try { + return new CreationResult<>(fc.get(), true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (ExecutionException e) { + log.debug(e.getMessage()); + return new CreationResult<>(null, false); + } + }) + .create(consumers); + } + @Override public CompletableFuture createConsumer( String topic, String subscriptionName, ConsumerCallback consumerCallback) { @@ -163,9 +218,6 @@ private Connection getOrCreateConnection(String primaryBrokerUri) { return connections.computeIfAbsent( primaryBrokerUri, p -> { - String[] userInfo = newURI(primaryBrokerUri).getUserInfo().split(":"); - String user = userInfo[0]; - String password = userInfo[1]; // RabbitMQ will pick the first available address from the list. Future reconnection // attempts will pick a random accessible address from the provided list. List
addresses = @@ -176,11 +228,15 @@ private Connection getOrCreateConnection(String primaryBrokerUri) { try { ConnectionFactory connectionFactory = new ConnectionFactory(); connectionFactory.setAutomaticRecoveryEnabled(true); - connectionFactory.setUsername(user); - connectionFactory.setPassword(password); + String userInfo = newURI(primaryBrokerUri).getUserInfo(); + if (userInfo != null) { + String[] userInfoElems = userInfo.split(":"); + connectionFactory.setUsername(userInfoElems[0]); + connectionFactory.setPassword(userInfoElems[1]); + } return connectionFactory.newConnection(addresses); } catch (Exception e) { - throw new RuntimeException("Couldn't establish connection", e); + throw new RuntimeException("Couldn't establish connection to: " + primaryBrokerUri, e); } }); } diff --git a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java index 40bd1cbb3..15a67b1bf 100644 --- a/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java +++ b/driver-rabbitmq/src/main/java/io/openmessaging/benchmark/driver/rabbitmq/RabbitMqConfig.java @@ -25,6 +25,10 @@ public class RabbitMqConfig { public List amqpUris = new ArrayList<>(); public boolean messagePersistence = false; public QueueType queueType = CLASSIC; + public long producerCreationDelay = 100; + public int producerCreationBatchSize = 5; + public long consumerCreationDelay = 100; + public int consumerCreationBatchSize = 5; public enum QueueType { CLASSIC { diff --git a/etc/findbugsExclude.xml b/etc/findbugsExclude.xml index 150f3a86c..a5af39dd8 100644 --- a/etc/findbugsExclude.xml +++ b/etc/findbugsExclude.xml @@ -126,4 +126,8 @@ + + + + From ed8276417cb0d7b58113a93e14c0271887d2edb5 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Wed, 25 Jan 2023 23:35:29 +0000 Subject: [PATCH 090/103] [Improvement] Pulsar 2.10.2 (#354) * Pulsar 2.10.2 * More version changes Co-authored-by: Matteo Merli --- driver-pulsar/deploy/ssd/deploy.yaml | 2 +- driver-pulsar/deploy/ssd/extra_vars.yaml | 2 +- driver-pulsar/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index f1242da04..cb90b51b8 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -23,7 +23,7 @@ serviceUrl: "pulsar://{{ hostvars[groups['pulsar'][0]].private_ip }}:6650/" httpUrl: "http://{{ hostvars[groups['pulsar'][0]].private_ip }}:8080/" - set_fact: - pulsar_version: "{{ pulsar_version | default('2.10.1') }}" + pulsar_version: "{{ pulsar_version | default('2.10.2') }}" node_exporter_version: "{{ node_exporter_version | default('1.2.2') }}" prometheus_version: "{{ prometheus_version | default('2.31.1') }}" disk_dev: "{{ disk_dev | default(['/dev/nvme1n1', '/dev/nvme2n1']) }}" diff --git a/driver-pulsar/deploy/ssd/extra_vars.yaml b/driver-pulsar/deploy/ssd/extra_vars.yaml index f6a9bd0a0..510a90fd9 100644 --- a/driver-pulsar/deploy/ssd/extra_vars.yaml +++ b/driver-pulsar/deploy/ssd/extra_vars.yaml @@ -12,7 +12,7 @@ # limitations under the License. # -pulsar_version: 2.10.1 +pulsar_version: 2.10.2 node_exporter_version: 1.2.2 prometheus_version: 2.31.1 diff --git a/driver-pulsar/pom.xml b/driver-pulsar/pom.xml index 865c4505a..934749838 100644 --- a/driver-pulsar/pom.xml +++ b/driver-pulsar/pom.xml @@ -25,7 +25,7 @@ driver-pulsar - 2.10.1 + 2.10.2 From defa46fe750c01ff28179d8d3ff06f050bd4bc08 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 1 Mar 2023 10:48:11 -0800 Subject: [PATCH 091/103] Pulsar 2.11.0 --- driver-pulsar/deploy/ssd/deploy.yaml | 6 +++--- driver-pulsar/deploy/ssd/extra_vars.yaml | 2 +- driver-pulsar/deploy/ssd/terraform.tfvars | 2 +- driver-pulsar/pom.xml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/driver-pulsar/deploy/ssd/deploy.yaml b/driver-pulsar/deploy/ssd/deploy.yaml index cb90b51b8..48a816dbb 100644 --- a/driver-pulsar/deploy/ssd/deploy.yaml +++ b/driver-pulsar/deploy/ssd/deploy.yaml @@ -23,7 +23,7 @@ serviceUrl: "pulsar://{{ hostvars[groups['pulsar'][0]].private_ip }}:6650/" httpUrl: "http://{{ hostvars[groups['pulsar'][0]].private_ip }}:8080/" - set_fact: - pulsar_version: "{{ pulsar_version | default('2.10.2') }}" + pulsar_version: "{{ pulsar_version | default('2.11.0') }}" node_exporter_version: "{{ node_exporter_version | default('1.2.2') }}" prometheus_version: "{{ prometheus_version | default('2.31.1') }}" disk_dev: "{{ disk_dev | default(['/dev/nvme1n1', '/dev/nvme2n1']) }}" @@ -416,12 +416,12 @@ lineinfile: dest: /opt/benchmark/bin/benchmark-worker regexp: '^JVM_MEM=' - line: 'JVM_MEM="-Xms100G -Xmx100G -XX:+UseZGC"' + line: 'JVM_MEM="-Xms64G -Xmx64G -XX:+UseZGC"' - name: Configure memory lineinfile: dest: /opt/benchmark/bin/benchmark regexp: '^JVM_MEM=' - line: 'JVM_MEM="-Xmx4G"' + line: 'JVM_MEM="-Xmx16G"' - name: Install benchmark systemd service template: src: "templates/benchmark-worker.service" diff --git a/driver-pulsar/deploy/ssd/extra_vars.yaml b/driver-pulsar/deploy/ssd/extra_vars.yaml index 510a90fd9..6aaa38db9 100644 --- a/driver-pulsar/deploy/ssd/extra_vars.yaml +++ b/driver-pulsar/deploy/ssd/extra_vars.yaml @@ -12,7 +12,7 @@ # limitations under the License. # -pulsar_version: 2.10.2 +pulsar_version: 2.11.0 node_exporter_version: 1.2.2 prometheus_version: 2.31.1 diff --git a/driver-pulsar/deploy/ssd/terraform.tfvars b/driver-pulsar/deploy/ssd/terraform.tfvars index bc4a9fe2e..7164caa2d 100644 --- a/driver-pulsar/deploy/ssd/terraform.tfvars +++ b/driver-pulsar/deploy/ssd/terraform.tfvars @@ -12,7 +12,7 @@ instance_types = { num_instances = { "client" = 4 - "pulsar" = 3 + "pulsar" = 5 "zookeeper" = 3 "prometheus" = 1 } diff --git a/driver-pulsar/pom.xml b/driver-pulsar/pom.xml index 934749838..5032b74f8 100644 --- a/driver-pulsar/pom.xml +++ b/driver-pulsar/pom.xml @@ -25,7 +25,7 @@ driver-pulsar - 2.10.2 + 2.11.0 From 5eebc5b58863962df8c44cc0c882de86109a5947 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:43:41 +0800 Subject: [PATCH 092/103] Bump testng from 7.6.1 to 7.7.0 in /driver-kop (#366) Bumps [testng](https://github.com/cbeust/testng) from 7.6.1 to 7.7.0. - [Release notes](https://github.com/cbeust/testng/releases) - [Changelog](https://github.com/cbeust/testng/blob/master/CHANGES.txt) - [Commits](https://github.com/cbeust/testng/compare/7.6.1...7.7.0) --- updated-dependencies: - dependency-name: org.testng:testng dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- driver-kop/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-kop/pom.xml b/driver-kop/pom.xml index 1904ed9d4..74843ac77 100644 --- a/driver-kop/pom.xml +++ b/driver-kop/pom.xml @@ -53,7 +53,7 @@ org.testng testng - 7.6.1 + 7.7.0 test From 6a947ebfb9e9eb5d8ec8582644eb9db410165653 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Sat, 12 Aug 2023 00:03:32 +0800 Subject: [PATCH 093/103] Fix failed drive-kop tests (#387) * Fix failed drive-kop tests * Fix format violation --- driver-kop/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/driver-kop/pom.xml b/driver-kop/pom.xml index 74843ac77..def80e60b 100644 --- a/driver-kop/pom.xml +++ b/driver-kop/pom.xml @@ -50,6 +50,11 @@ org.apache.commons commons-lang3 + + org.apache.logging.log4j + log4j-slf4j-impl + test + org.testng testng From 31215cca5bf81ed09eb3edce5f1e13a2b20167b1 Mon Sep 17 00:00:00 2001 From: Sky Kistler Date: Fri, 11 Aug 2023 12:07:26 -0400 Subject: [PATCH 094/103] Bump log4j from 2.17.1 to 2.20.0 (#374) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5542f673b..03ff8b9d5 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 4.14.4 10.3.3 3.12.0 - 2.17.1 + 2.20.0 1.18.24 2.13.2 1.48 From 1c681e9f06d7d6465c70b8baa79e8af320393a4e Mon Sep 17 00:00:00 2001 From: Sky Kistler Date: Fri, 11 Aug 2023 12:12:23 -0400 Subject: [PATCH 095/103] Add maven exclusion for netty transitive dependency in artemis (#376) * exclude artemis netty transitive dependency * lint pom.xml --------- Co-authored-by: sky.kistler --- benchmark-framework/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index 256222f9d..247f97b83 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -39,6 +39,12 @@ ${project.groupId} driver-artemis ${project.version} + + + io.netty + * + + ${project.groupId} From b3a90448c1b8b02d9baecad660118b9201d75ea7 Mon Sep 17 00:00:00 2001 From: kevin-ip Date: Mon, 28 Aug 2023 13:36:13 -0500 Subject: [PATCH 096/103] [NATS] Extend to support in-memory stream (#370) * [nats] Add storage type to config so one can test with file or memory stream. * [nats] Add maxBytes to config so one can prevent NATS running out of memory incase the storage type is set to "Memory" --- .../benchmark/driver/nats/NatsBenchmarkDriver.java | 4 ++-- .../openmessaging/benchmark/driver/nats/NatsConfig.java | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java index c5b677f30..8071f36df 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsBenchmarkDriver.java @@ -28,7 +28,6 @@ import io.nats.client.Options; import io.nats.client.PushSubscribeOptions; import io.nats.client.api.ConsumerConfiguration; -import io.nats.client.api.StorageType; import io.nats.client.api.StreamConfiguration; import io.nats.client.api.StreamInfo; import io.nats.client.support.JsonUtils; @@ -94,7 +93,8 @@ public CompletableFuture createTopic(String topic, int partitions) { StreamConfiguration.builder() .name(topic) .subjects(topic) - .storageType(StorageType.File) + .storageType(config.storageType) + .maxBytes(config.maxBytes) .replicas(config.replicationFactor) .build()); log.info("Created stream {} -- {}", topic, JsonUtils.getFormatted(streamInfo)); diff --git a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java index f6770c982..d76a900b3 100644 --- a/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java +++ b/driver-nats/src/main/java/io/openmessaging/benchmark/driver/nats/NatsConfig.java @@ -13,8 +13,16 @@ */ package io.openmessaging.benchmark.driver.nats; + +import io.nats.client.api.StorageType; + public class NatsConfig { public String natsHostUrl; public int replicationFactor; + + public StorageType storageType = StorageType.File; + + // -1 is unlimited + public int maxBytes = -1; } From f2a4e8cae6fe294a6af7abb677c2801dc81bc73e Mon Sep 17 00:00:00 2001 From: filipe oliveira Date: Thu, 7 Sep 2023 01:27:09 +0100 Subject: [PATCH 097/103] Bumped Jedis to v5.0.0 - fixed breaking changes from v3.7.0 (#390) --- driver-redis/pom.xml | 2 +- .../benchmark/driver/redis/RedisBenchmarkConsumer.java | 2 +- .../benchmark/driver/redis/RedisBenchmarkDriver.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/driver-redis/pom.xml b/driver-redis/pom.xml index 38c9a2855..32f877b1b 100644 --- a/driver-redis/pom.xml +++ b/driver-redis/pom.xml @@ -38,7 +38,7 @@ redis.clients jedis - 3.7.0 + 5.0.0 diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java index 877d12080..4e8bb88e8 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkConsumer.java @@ -27,9 +27,9 @@ import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.StreamEntry; import redis.clients.jedis.StreamEntryID; import redis.clients.jedis.params.XReadGroupParams; +import redis.clients.jedis.resps.StreamEntry; public class RedisBenchmarkConsumer implements BenchmarkConsumer { private final JedisPool pool; diff --git a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java index 2579ccb30..d1ef95c3d 100644 --- a/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java +++ b/driver-redis/src/main/java/io/openmessaging/benchmark/driver/redis/RedisBenchmarkDriver.java @@ -28,11 +28,11 @@ import java.util.Random; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; public class RedisBenchmarkDriver implements BenchmarkDriver { JedisPool jedisPool; @@ -80,7 +80,7 @@ public CompletableFuture createConsumer( } private void setupJedisConn() { - JedisPoolConfig poolConfig = new JedisPoolConfig(); + GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig<>(); poolConfig.setMaxTotal(this.clientConfig.jedisPoolMaxTotal); poolConfig.setMaxIdle(this.clientConfig.jedisPoolMaxIdle); if (this.clientConfig.redisPass != null) { From 0a68a615e3f39efb5a1b55ac7bcb9304d5fff9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Gracia?= Date: Tue, 26 Sep 2023 18:17:30 +0200 Subject: [PATCH 098/103] Issue 391: Update Pravega library versions and deployment script (#392) --- benchmark-framework/pom.xml | 11 ++-- driver-pravega/README.md | 24 ++++---- driver-pravega/deploy/deploy.yaml | 61 +++++++++++-------- .../deploy/provision-pravega-aws.tf | 2 +- driver-pravega/deploy/terraform.tfvars | 10 +-- driver-pravega/deploy/vars.yaml | 2 +- driver-pravega/pom.xml | 4 +- 7 files changed, 64 insertions(+), 50 deletions(-) diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index 247f97b83..0e324141f 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -71,11 +71,6 @@ driver-nats ${project.version} - - ${project.groupId} - driver-nats-streaming - ${project.version} - ${project.groupId} driver-nsq @@ -145,6 +140,12 @@ HdrHistogram 2.1.12 + + ${project.groupId} + driver-nats-streaming + ${project.version} + provided + org.projectlombok lombok diff --git a/driver-pravega/README.md b/driver-pravega/README.md index e67d9ba21..5b45bb6f9 100644 --- a/driver-pravega/README.md +++ b/driver-pravega/README.md @@ -30,13 +30,15 @@ check [how to build Pravega](doc/build_pravega.md). # DEPLOY A PRAVEGA CLUSTER ON AMAZON WEB SERVICES -You can deploy a Pravega cluster on AWS (for benchmarking purposes) using [Terraform 0.12.20](https://www.terraform.io/) and [Ansible 2.8.5](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html). -You’ll need to have both of those tools installed as well as the `terraform-inventory` [plugin](https://github.com/adammck/terraform-inventory) for Terraform. +You can deploy a Pravega cluster on AWS (for benchmarking purposes) using [Terraform 0.12.20](https://www.terraform.io/) +and [Ansible 2.8.5](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) (with a +a version of `Jinja2=<3.0.3`). +You’ll need to have both tools installed, as well as the `terraform-inventory` [plugin](https://github.com/adammck/terraform-inventory) for Terraform. -You also need to install an Ansible modules to support metrics. +You also need to install an Ansible modules to support metrics: ``` -ansible-galaxy install cloudalchemy.node-exporter +ansible-galaxy install cloudalchemy.node_exporter ``` In addition, you’ll need to: @@ -74,7 +76,7 @@ This will install the following [EC2](https://aws.amazon.com/ec2) instances (plu | Resource | Description | Count | |----------------------|-------------------------------------------------------------|-------| | Controller instances | The VMs on which a Pravega controller will run | 1 | -| Bookkeeper instances | The VMs on which a Bookkeeper and Segmentstore will run | 3 | +| Bookkeeper instances | The VMs on which a Bookkeeper and Segment Store will run | 3 | | ZooKeeper instances | The VMs on which a ZooKeeper node will run | 3 | | Client instance | The VM from which the benchmarking suite itself will be run | 2 | @@ -84,12 +86,12 @@ When you run `terraform apply`, you will be prompted to type `yes`. Type `yes` t There’s a handful of configurable parameters related to the Terraform deployment that you can alter by modifying the defaults in the `terraform.tfvars` file. -| Variable | Description | Default | -|-------------------|--------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| -| `region` | The AWS region in which the Pravega cluster will be deployed | `us-west-2` | -| `public_key_path` | The path to the SSH public key that you’ve generated | `~/.ssh/pravega_aws.pub` | -| `ami` | The [Amazon Machine Image (AWI)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) to be used by the cluster’s machines | `ami-9fa343e7` | -| `instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (BookKeeper bookies), `m5.large`(Controller), `t3.small` (ZooKeeper), `c5.4xlarge` (benchmarking client) | +| Variable | Description | Default | +|-------------------|--------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| `region` | The AWS region in which the Pravega cluster will be deployed | `us-east-2` | +| `public_key_path` | The path to the SSH public key that you’ve generated | `~/.ssh/pravega_aws.pub` | +| `ami` | The [Amazon Machine Image (AWI)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) to be used by the cluster’s machines | `ami-0bb2449c2217cb9b0` | +| `instance_types` | The EC2 instance types used by the various components | `i3en.2xlarge` (Segment Store + Bookkeeper), `m5n.xlarge`(Controller), `t2.small` (ZooKeeper), `m5n.xlarge` (benchmarking client), `t2.large` (metrics) | If you modify the `public_key_path`, make sure that you point to the appropriate SSH key path when running the [Ansible playbook](#_RUNNING_THE_ANSIBLE_PLAYBOOK). diff --git a/driver-pravega/deploy/deploy.yaml b/driver-pravega/deploy/deploy.yaml index 3846fa997..5aebfe169 100644 --- a/driver-pravega/deploy/deploy.yaml +++ b/driver-pravega/deploy/deploy.yaml @@ -98,20 +98,6 @@ echo 'LANG=en_US.utf-8 LC_ALL=en_US.utf-8' > /etc/environment -#- name: Install nmon -# hosts: ["!tier2"] -# tags: ["nmon"] -# connection: ssh -# become: true -# tasks: -# - name: Download nmon -# unarchive: -# src: "http://sourceforge.net/projects/nmon/files/nmon16j.tar.gz" -# remote_src: yes -# dest: /tmp -# - command: cp /tmp/nmon_AMD64_ubuntu1804 /usr/local/bin/nmon -# - command: chmod a+x /usr/local/bin/nmon - - name: Metrics installation hosts: ["metrics"] tags: ["metrics"] @@ -120,13 +106,38 @@ tasks: - name: Add Extras Repo shell: yum-config-manager --enable rhui-REGION-rhel-server-extras - - name: Install RPM packages - yum: pkg={{ item }} state=latest - with_items: - - docker - - systemd: + when: + - ansible_facts['distribution'] == 'RedHat' + - ansible_facts['distribution_major_version'] | int <= 7 + - name: Docker repo + yum_repository: + name: docker + description: repo for docker + baseurl: "https://download.docker.com/linux/centos/{{ ansible_facts['distribution_major_version'] }}/x86_64/stable/" + gpgcheck: no + when: ansible_facts['distribution'] == 'RedHat' + - name: Create Docker repo dir + file: + path: "/etc/yum.repos.d/" + state: directory + - name: Add Docker Centos extras + copy: + dest: /etc/yum.repos.d/docker-ce.repo + content: | + [centos-extras] + name=Centos extras - $basearch + baseurl=http://mirror.centos.org/centos/7/extras/x86_64 + enabled=1 + gpgcheck=1 + gpgkey=http://centos.org/keys/RPM-GPG-KEY-CentOS-7 + - name: Installing docker + yum: + state: latest + pkg: ['slirp4netns', 'fuse-overlayfs', 'container-selinux', 'docker-ce'] + - name: Start docker + service: + name: docker state: started - name: "docker" enabled: yes - name: Prometheus installation @@ -192,7 +203,7 @@ hosts: ["!tier2"] tags: ["node-exporter"] roles: - - cloudalchemy.node-exporter + - cloudalchemy.node_exporter - name: ZooKeeper setup hosts: zookeeper @@ -304,7 +315,7 @@ path: "{{ item.path }}" state: unmounted with_items: - - { path: "/mnt/journal", src: "/dev/nvme0n1" } + - { path: "/mnt/journal", src: "/dev/nvme2n1" } - { path: "/mnt/storage", src: "/dev/nvme1n1" } - name: Format disks filesystem: @@ -312,7 +323,7 @@ dev: '{{ item }}' force: yes with_items: - - '/dev/nvme0n1' + - '/dev/nvme2n1' - '/dev/nvme1n1' - name: Mount disks mount: @@ -322,7 +333,7 @@ opts: defaults,noatime,nodiscard state: mounted with_items: - - { path: "/mnt/journal", src: "/dev/nvme0n1" } + - { path: "/mnt/journal", src: "/dev/nvme2n1" } - { path: "/mnt/storage", src: "/dev/nvme1n1" } - name: BookKeeper setup @@ -350,7 +361,7 @@ dest: "/opt/bookkeeper/bin/common.sh" - name: Format BookKeeper metadata in Zookeeper command: > - bin/bookkeeper shell metaformat -nonInteractive --force + bin/bookkeeper shell initnewcluster args: chdir: /opt/bookkeeper when: groups['bookkeeper'][0] == inventory_hostname diff --git a/driver-pravega/deploy/provision-pravega-aws.tf b/driver-pravega/deploy/provision-pravega-aws.tf index 49f959a70..f3c4b6236 100644 --- a/driver-pravega/deploy/provision-pravega-aws.tf +++ b/driver-pravega/deploy/provision-pravega-aws.tf @@ -190,7 +190,7 @@ resource "aws_instance" "metrics" { # Change the EFS provisioned TP here resource "aws_efs_file_system" "tier2" { throughput_mode = "provisioned" - provisioned_throughput_in_mibps = 1000 + provisioned_throughput_in_mibps = 100 tags = { Name = "pravega-tier2" } diff --git a/driver-pravega/deploy/terraform.tfvars b/driver-pravega/deploy/terraform.tfvars index 52c9e863e..ecac89770 100644 --- a/driver-pravega/deploy/terraform.tfvars +++ b/driver-pravega/deploy/terraform.tfvars @@ -1,12 +1,12 @@ public_key_path = "~/.ssh/pravega_aws.pub" -region = "us-west-2" -ami = "ami-9fa343e7" // RHEL-7.4 us-west-2 +region = "us-east-2" +ami = "ami-0bb2449c2217cb9b0" // RHEL-7.9 us-east-2 instance_types = { "controller" = "m5.large" - "bookkeeper" = "i3en.6xlarge" + "bookkeeper" = "i3en.2xlarge" "zookeeper" = "t2.small" - "client" = "m5n.8xlarge" + "client" = "m5n.xlarge" "metrics" = "t2.large" } @@ -14,6 +14,6 @@ num_instances = { "controller" = 1 "bookkeeper" = 3 "zookeeper" = 3 - "client" = 2 + "client" = 1 "metrics" = 1 } diff --git a/driver-pravega/deploy/vars.yaml b/driver-pravega/deploy/vars.yaml index 730eb107d..e6e9543aa 100644 --- a/driver-pravega/deploy/vars.yaml +++ b/driver-pravega/deploy/vars.yaml @@ -13,7 +13,7 @@ # --- -pravegaVersion: "0.10.1" +pravegaVersion: "0.12.0" zookeeperVersion: "3.5.5" bookkeeperVersion: "4.14.2" prometheusVersion: "2.2.1" diff --git a/driver-pravega/pom.xml b/driver-pravega/pom.xml index 5cce48a41..d1d184f9e 100644 --- a/driver-pravega/pom.xml +++ b/driver-pravega/pom.xml @@ -27,7 +27,7 @@ - 0.10.2 + 0.12.0 @@ -64,7 +64,7 @@ io.pravega pravega-keycloak-client - 0.11.0 + ${pravegaVersion} org.apache.commons From 71b6ff2482316e9accd74dbd93f5bff322a1feba Mon Sep 17 00:00:00 2001 From: guyinyou Date: Thu, 16 Nov 2023 19:07:41 +0800 Subject: [PATCH 099/103] update rocketmq engine --- driver-rocketmq/pom.xml | 2 +- driver-rocketmq/rocketmq.yaml | 10 ++++ .../rocketmq/RocketMQBenchmarkDriver.java | 55 ++++++++++++++++++- .../rocketmq/client/RocketMQClientConfig.java | 8 +++ 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/driver-rocketmq/pom.xml b/driver-rocketmq/pom.xml index 9e60ff3cc..3cd0a9139 100644 --- a/driver-rocketmq/pom.xml +++ b/driver-rocketmq/pom.xml @@ -26,7 +26,7 @@ driver-rocketmq - 4.9.3 + 5.1.4 diff --git a/driver-rocketmq/rocketmq.yaml b/driver-rocketmq/rocketmq.yaml index b5aec62ee..b9b608088 100644 --- a/driver-rocketmq/rocketmq.yaml +++ b/driver-rocketmq/rocketmq.yaml @@ -19,5 +19,15 @@ driverClass: io.openmessaging.benchmark.driver.rocketmq.RocketMQBenchmarkDriver clusterName: DefaultCluster namesrvAddr: 127.0.0.1:9876 vipChannelEnabled: false + +batchCQ: true +autoBatch: true +# batchMaxBytes: 32768 +# batchMaxDelayMs: 10 +# totalBatchMaxBytes: 33554432 + +enableBackpressure: true +backpressureConcurrency: 1024 + accessKey: secretKey: \ No newline at end of file diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java index d7d547098..d5e2edeb9 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java @@ -26,9 +26,11 @@ import io.openmessaging.benchmark.driver.rocketmq.client.RocketMQClientConfig; import java.io.File; import java.io.IOException; +import java.util.Map; import java.util.Random; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.acl.common.AclClientRPCHook; @@ -37,12 +39,18 @@ import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.TopicAttributes; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.admin.MQAdminExt; import org.apache.rocketmq.tools.command.CommandUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,6 +88,28 @@ public String getTopicNamePrefix() { return "RocketMQ-Benchmark"; } + Map> cachedBrokerAddr = new ConcurrentHashMap<>(); + + int fetchCnt = 0; + + private synchronized Set fetchMasterAndSlaveAddrByClusterName(final MQAdminExt adminExt, + final String clusterName) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + Set brokerList = cachedBrokerAddr.get(clusterName); + if (brokerList == null) { + brokerList = + CommandUtil.fetchMasterAndSlaveAddrByClusterName( + adminExt, this.rmqClientConfig.clusterName); + cachedBrokerAddr.put(clusterName, brokerList); + if (brokerList.isEmpty()) { + throw new RuntimeException("get brokerAddr return null, clusterName: " + clusterName); + } + } + if (fetchCnt++ % 100 == 0) { + log.info("fetch brokerAddr count: " + fetchCnt); + } + return brokerList; + } + @Override public CompletableFuture createTopic(final String topic, final int partitions) { return CompletableFuture.runAsync( @@ -90,10 +120,13 @@ public CompletableFuture createTopic(final String topic, final int partiti topicConfig.setReadQueueNums(partitions); topicConfig.setWriteQueueNums(partitions); topicConfig.setTopicName(topic); + if (Boolean.TRUE.equals(this.rmqClientConfig.batchCQ)) { + topicConfig.getAttributes().put("+" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), "BatchCQ"); + } try { Set brokerList = - CommandUtil.fetchMasterAddrByClusterName( + fetchMasterAndSlaveAddrByClusterName( this.rmqAdmin, this.rmqClientConfig.clusterName); topicConfig.setReadQueueNums(Math.max(1, partitions / brokerList.size())); topicConfig.setWriteQueueNums(Math.max(1, partitions / brokerList.size())); @@ -130,6 +163,26 @@ public CompletableFuture createProducer(final String topic) { if (null != this.rmqClientConfig.compressMsgBodyOverHowmuch) { rmqProducer.setCompressMsgBodyOverHowmuch(this.rmqClientConfig.compressMsgBodyOverHowmuch); } + + if (null != this.rmqClientConfig.autoBatch) { + rmqProducer.setAutoBatch(this.rmqClientConfig.autoBatch); + } + if (null != this.rmqClientConfig.batchMaxBytes) { + rmqProducer.batchMaxBytes(this.rmqClientConfig.batchMaxBytes); + } + if (null != this.rmqClientConfig.batchMaxDelayMs) { + rmqProducer.batchMaxDelayMs(this.rmqClientConfig.batchMaxDelayMs); + } + if (null != this.rmqClientConfig.totalBatchMaxBytes) { + rmqProducer.totalBatchMaxBytes(this.rmqClientConfig.totalBatchMaxBytes); + } + if (null != this.rmqClientConfig.enableBackpressure) { + rmqProducer.setEnableBackpressureForAsyncMode(this.rmqClientConfig.enableBackpressure); + } + if (null != this.rmqClientConfig.backpressureConcurrency) { + rmqProducer.setBackPressureForAsyncSendNum(this.rmqClientConfig.backpressureConcurrency); + } + try { rmqProducer.start(); } catch (MQClientException e) { diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java index 840090452..26cb109bb 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/client/RocketMQClientConfig.java @@ -19,6 +19,14 @@ public class RocketMQClientConfig { public Boolean vipChannelEnabled; public Integer maxMessageSize; public Integer compressMsgBodyOverHowmuch; + public Boolean batchCQ; + public Boolean autoBatch; + public Integer batchMaxBytes; + public Integer batchMaxDelayMs; + public Integer totalBatchMaxBytes; + + public Boolean enableBackpressure; + public Integer backpressureConcurrency; public String accessKey; public String secretKey; } From d0c0277fec37379a466e41e2917f0765901d6f9c Mon Sep 17 00:00:00 2001 From: guyinyou Date: Thu, 16 Nov 2023 19:52:38 +0800 Subject: [PATCH 100/103] fix codestyle --- .../rocketmq/RocketMQBenchmarkDriver.java | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java index d5e2edeb9..cc54c7577 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java @@ -93,7 +93,8 @@ public String getTopicNamePrefix() { int fetchCnt = 0; private synchronized Set fetchMasterAndSlaveAddrByClusterName(final MQAdminExt adminExt, - final String clusterName) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + final String clusterName) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQBrokerException, InterruptedException { Set brokerList = cachedBrokerAddr.get(clusterName); if (brokerList == null) { brokerList = @@ -113,35 +114,35 @@ private synchronized Set fetchMasterAndSlaveAddrByClusterName(final MQAd @Override public CompletableFuture createTopic(final String topic, final int partitions) { return CompletableFuture.runAsync( - () -> { - TopicConfig topicConfig = new TopicConfig(); - topicConfig.setOrder(false); - topicConfig.setPerm(6); - topicConfig.setReadQueueNums(partitions); - topicConfig.setWriteQueueNums(partitions); - topicConfig.setTopicName(topic); - if (Boolean.TRUE.equals(this.rmqClientConfig.batchCQ)) { - topicConfig.getAttributes().put("+" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), "BatchCQ"); - } + () -> { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setOrder(false); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(partitions); + topicConfig.setWriteQueueNums(partitions); + topicConfig.setTopicName(topic); + if (Boolean.TRUE.equals(this.rmqClientConfig.batchCQ)) { + topicConfig.getAttributes().put("+" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), "BatchCQ"); + } - try { - Set brokerList = - fetchMasterAndSlaveAddrByClusterName( - this.rmqAdmin, this.rmqClientConfig.clusterName); - topicConfig.setReadQueueNums(Math.max(1, partitions / brokerList.size())); - topicConfig.setWriteQueueNums(Math.max(1, partitions / brokerList.size())); + try { + Set brokerList = + fetchMasterAndSlaveAddrByClusterName( + this.rmqAdmin, this.rmqClientConfig.clusterName); + topicConfig.setReadQueueNums(Math.max(1, partitions / brokerList.size())); + topicConfig.setWriteQueueNums(Math.max(1, partitions / brokerList.size())); - for (String brokerAddr : brokerList) { - this.rmqAdmin.createAndUpdateTopicConfig(brokerAddr, topicConfig); - } - } catch (Exception e) { - throw new RuntimeException( - String.format( - "Failed to create topic [%s] to cluster [%s]", - topic, this.rmqClientConfig.clusterName), - e); + for (String brokerAddr : brokerList) { + this.rmqAdmin.createAndUpdateTopicConfig(brokerAddr, topicConfig); } - }); + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to create topic [%s] to cluster [%s]", + topic, this.rmqClientConfig.clusterName), + e); + } + }); } @Override From f423781df93275edbeeded603f41b5ce1ca7be51 Mon Sep 17 00:00:00 2001 From: StevenLuMT Date: Fri, 15 Dec 2023 00:49:34 +0800 Subject: [PATCH 101/103] Fixed for 'Failed to execute goal com.diffplug.spotless:spotless-maven-plugin:2.25.0:check (check) on project driver-rocketmq: The following files had format violations: [ERROR] src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java' (#399) Co-authored-by: nuolin --- .../rocketmq/RocketMQBenchmarkDriver.java | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java index cc54c7577..ccad035fd 100644 --- a/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java +++ b/driver-rocketmq/src/main/java/io/openmessaging/benchmark/driver/rocketmq/RocketMQBenchmarkDriver.java @@ -92,14 +92,15 @@ public String getTopicNamePrefix() { int fetchCnt = 0; - private synchronized Set fetchMasterAndSlaveAddrByClusterName(final MQAdminExt adminExt, - final String clusterName) throws RemotingConnectException, RemotingSendRequestException, - RemotingTimeoutException, MQBrokerException, InterruptedException { + private synchronized Set fetchMasterAndSlaveAddrByClusterName( + final MQAdminExt adminExt, final String clusterName) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + MQBrokerException, InterruptedException { Set brokerList = cachedBrokerAddr.get(clusterName); if (brokerList == null) { brokerList = - CommandUtil.fetchMasterAndSlaveAddrByClusterName( - adminExt, this.rmqClientConfig.clusterName); + CommandUtil.fetchMasterAndSlaveAddrByClusterName( + adminExt, this.rmqClientConfig.clusterName); cachedBrokerAddr.put(clusterName, brokerList); if (brokerList.isEmpty()) { throw new RuntimeException("get brokerAddr return null, clusterName: " + clusterName); @@ -114,35 +115,37 @@ private synchronized Set fetchMasterAndSlaveAddrByClusterName(final MQAd @Override public CompletableFuture createTopic(final String topic, final int partitions) { return CompletableFuture.runAsync( - () -> { - TopicConfig topicConfig = new TopicConfig(); - topicConfig.setOrder(false); - topicConfig.setPerm(6); - topicConfig.setReadQueueNums(partitions); - topicConfig.setWriteQueueNums(partitions); - topicConfig.setTopicName(topic); - if (Boolean.TRUE.equals(this.rmqClientConfig.batchCQ)) { - topicConfig.getAttributes().put("+" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), "BatchCQ"); - } + () -> { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setOrder(false); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(partitions); + topicConfig.setWriteQueueNums(partitions); + topicConfig.setTopicName(topic); + if (Boolean.TRUE.equals(this.rmqClientConfig.batchCQ)) { + topicConfig + .getAttributes() + .put("+" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), "BatchCQ"); + } - try { - Set brokerList = - fetchMasterAndSlaveAddrByClusterName( - this.rmqAdmin, this.rmqClientConfig.clusterName); - topicConfig.setReadQueueNums(Math.max(1, partitions / brokerList.size())); - topicConfig.setWriteQueueNums(Math.max(1, partitions / brokerList.size())); + try { + Set brokerList = + fetchMasterAndSlaveAddrByClusterName( + this.rmqAdmin, this.rmqClientConfig.clusterName); + topicConfig.setReadQueueNums(Math.max(1, partitions / brokerList.size())); + topicConfig.setWriteQueueNums(Math.max(1, partitions / brokerList.size())); - for (String brokerAddr : brokerList) { - this.rmqAdmin.createAndUpdateTopicConfig(brokerAddr, topicConfig); + for (String brokerAddr : brokerList) { + this.rmqAdmin.createAndUpdateTopicConfig(brokerAddr, topicConfig); + } + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to create topic [%s] to cluster [%s]", + topic, this.rmqClientConfig.clusterName), + e); } - } catch (Exception e) { - throw new RuntimeException( - String.format( - "Failed to create topic [%s] to cluster [%s]", - topic, this.rmqClientConfig.clusterName), - e); - } - }); + }); } @Override From 4eb2327c0205108678887c1f511a4fa1d460d5ac Mon Sep 17 00:00:00 2001 From: Travis Downs Date: Tue, 19 Dec 2023 22:36:32 -0300 Subject: [PATCH 102/103] Grow buffer if histogram can't fit (#398) * Grow the histogram buffer if it is too small When we serialize a histogram to a byte array, the intermediate ByteBuffer that we pass may be too small which may result in silent truncation of the encoded histogram. This will manifest on the driver side as a decoding failure. This change detects this case and grows the buffer by a factor of 2 until it fits. Fixes openmessaging/benchmark#369. * Add Histogram deserialization test Add an addition test to the HistogramSerDeTest which tests that histogram deserialization roundtrips even when the serialized size exceeds the initial buffer. * Fixed spotbugs warnings --------- Co-authored-by: Matteo Merli --- .../worker/jackson/HistogramSerializer.java | 37 +++++++++-- .../worker/jackson/HistogramSerDeTest.java | 62 +++++++++++++++++++ 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java index a1b61afa3..c1c253f16 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/jackson/HistogramSerializer.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.google.common.base.Preconditions; import java.io.IOException; import java.nio.ByteBuffer; import org.HdrHistogram.Histogram; @@ -30,16 +31,40 @@ public HistogramSerializer() { super(Histogram.class); } + static byte[] toByteArray(ByteBuffer buffer) { + byte[] encodedBuffer = new byte[buffer.remaining()]; + buffer.get(encodedBuffer); + return encodedBuffer; + } + + static ByteBuffer serializeHistogram(Histogram histo, ByteBuffer buffer) { + buffer.clear(); + while (true) { + final int outBytes = histo.encodeIntoCompressedByteBuffer(buffer); + Preconditions.checkState(outBytes == buffer.position()); + final int capacity = buffer.capacity(); + if (outBytes < capacity) { + // encoding succesful + break; + } + // We filled the entire buffer, an indication that the buffer was not + // large enough, so we double the buffer and try again. + // See: https://github.com/HdrHistogram/HdrHistogram/issues/201 + buffer = ByteBuffer.allocate(capacity * 2); + } + buffer.flip(); + return buffer; + } + @Override public void serialize( Histogram histogram, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { ByteBuffer buffer = threadBuffer.get(); - buffer.clear(); - histogram.encodeIntoCompressedByteBuffer(buffer); - byte[] arr = new byte[buffer.position()]; - buffer.flip(); - buffer.get(arr); - jsonGenerator.writeBinary(arr); + ByteBuffer newBuffer = serializeHistogram(histogram, buffer); + if (newBuffer != buffer) { + threadBuffer.set(newBuffer); + } + jsonGenerator.writeBinary(toByteArray(newBuffer)); } } diff --git a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java index b06ef7566..9769eac6f 100644 --- a/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java +++ b/benchmark-framework/src/test/java/io/openmessaging/benchmark/worker/jackson/HistogramSerDeTest.java @@ -18,6 +18,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Random; import org.HdrHistogram.Histogram; import org.junit.jupiter.api.Test; @@ -41,4 +43,64 @@ void deserialize() throws IOException { assertThat(deserialized).isEqualTo(value); } + + /** + * Create a random histogram and insert the given number of samples. + * + * @param samples the number of samples to record into the histogram + * @return a Histogram with the given number of samples + */ + private Histogram randomHisto(int samples) { + Random r = new Random(0xBADBEEF); + Histogram h = new org.HdrHistogram.Histogram(5); + for (int i = 0; i < samples; i++) { + h.recordValue(r.nextInt(10000000)); + } + + return h; + } + + byte[] serializeRandomHisto(int samples, int initialBufferSize) throws Exception { + ByteBuffer inbuffer = ByteBuffer.allocate(initialBufferSize); + Histogram inHisto = randomHisto(samples); + byte[] serialBytes = + HistogramSerializer.toByteArray(HistogramSerializer.serializeHistogram(inHisto, inbuffer)); + + // check roundtrip + Histogram outHisto = + Histogram.decodeFromCompressedByteBuffer(ByteBuffer.wrap(serialBytes), Long.MIN_VALUE); + assertThat(inHisto).isEqualTo(outHisto); + + return serialBytes; + } + + @Test + public void testHistogram() throws Exception { + + // in the worker it's 8 MB but it takes a while to make a histogram that big + final int bufSize = 1002; + + int samples = 300; + + // we do an exponential search to fit the crossover point where we need to grow the buffer + while (true) { + byte[] serialBytes = serializeRandomHisto(samples, bufSize); + // System.out.println("Samples: " + samples + ", histogram size: " + serialBytes.length); + if (serialBytes.length >= bufSize) { + break; + } + samples *= 1.05; + } + + // then walk backwards across the point linearly with increment of 1 to check the boundary + // carefully + while (true) { + samples--; + byte[] serialBytes = serializeRandomHisto(samples, bufSize); + // System.out.println("Samples: " + samples + ", histogram size: " + serialBytes.length); + if (serialBytes.length < bufSize - 10) { + break; + } + } + } } From 13785cab9d899efc70990b90fb3432d048296774 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:36:45 -0800 Subject: [PATCH 103/103] Bump com.rabbitmq:amqp-client from 4.8.0 to 5.18.0 in /driver-rabbitmq (#394) Bumps [com.rabbitmq:amqp-client](https://github.com/rabbitmq/rabbitmq-java-client) from 4.8.0 to 5.18.0. - [Release notes](https://github.com/rabbitmq/rabbitmq-java-client/releases) - [Commits](https://github.com/rabbitmq/rabbitmq-java-client/compare/v4.8.0...v5.18.0) --- updated-dependencies: - dependency-name: com.rabbitmq:amqp-client dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- driver-rabbitmq/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-rabbitmq/pom.xml b/driver-rabbitmq/pom.xml index d92646d1d..6d1001bdf 100644 --- a/driver-rabbitmq/pom.xml +++ b/driver-rabbitmq/pom.xml @@ -33,7 +33,7 @@ com.rabbitmq amqp-client - 4.8.0 + 5.18.0 io.netty