diff --git a/.bazelrc b/.bazelrc index 7333057fba1..b0c5e1695af 100644 --- a/.bazelrc +++ b/.bazelrc @@ -24,6 +24,7 @@ build --enable_platform_specific_config test --action_env=TEST_TMPDIR=/tmp test --experimental_strict_java_deps=warn +test --experimental_ui_max_stdouterr_bytes=10485760 build --experimental_strict_java_deps=warn test --test_output=errors diff --git a/README.md b/README.md index 56d253ce1f4..5aaa2ba73c3 100644 --- a/README.md +++ b/README.md @@ -49,21 +49,21 @@ $ java -version java version "1.8.0_121" ``` -For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip) to download the 5.1.3 RocketMQ binary release, +For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip) to download the 5.1.4 RocketMQ binary release, unpack it to your local disk, such as `D:\rocketmq`. For macOS and Linux users, execute following commands: ```shell # Download release from the Apache mirror -$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip +$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip # Unpack the release -$ unzip rocketmq-all-5.1.3-bin-release.zip +$ unzip rocketmq-all-5.1.4-bin-release.zip ``` Prepare a terminal and change to the extracted `bin` directory: ```shell -$ cd rocketmq-all-5.1.3-bin-release/bin +$ cd rocketmq-all-5.1.4-bin-release/bin ``` **1) Start NameServer** diff --git a/WORKSPACE b/WORKSPACE index 3126f2d1d5d..c749ba2f26c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -92,6 +92,7 @@ maven_install( "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha", "io.opentelemetry:opentelemetry-exporter-logging:1.29.0", "io.opentelemetry:opentelemetry-sdk:1.29.0", + "io.opentelemetry:opentelemetry-exporter-logging-otlp:1.29.0", "com.squareup.okio:okio-jvm:3.0.0", "io.opentelemetry:opentelemetry-api:1.29.0", "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0", @@ -105,7 +106,7 @@ maven_install( "com.fasterxml.jackson.core:jackson-databind:2.13.4.2", "com.adobe.testing:s3mock-junit4:2.11.0", "io.github.aliyunmq:rocketmq-grpc-netty-codec-haproxy:1.0.0", - "io.github.aliyunmq:rocketmq-rocksdb:1.0.3", + "org.apache.rocketmq:rocketmq-rocksdb:1.0.2", ], fetch_sources = True, repositories = [ diff --git a/acl/pom.xml b/acl/pom.xml index 989c0cf77f7..8a296e5ae9e 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT rocketmq-acl rocketmq-acl ${project.version} diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel index 6adcdc7b99c..ab413d3d060 100644 --- a/broker/BUILD.bazel +++ b/broker/BUILD.bazel @@ -44,6 +44,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", "@maven//:io_opentelemetry_opentelemetry_exporter_logging", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:io_opentelemetry_opentelemetry_sdk", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", @@ -53,7 +54,7 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_logback_classic", "@maven//:org_slf4j_jul_to_slf4j", "@maven//:io_github_aliyunmq_rocketmq_shaded_slf4j_api_bridge", - "@maven//:io_github_aliyunmq_rocketmq_rocksdb", + "@maven//:org_apache_rocketmq_rocketmq_rocksdb", "@maven//:net_java_dev_jna_jna", ], ) diff --git a/broker/pom.xml b/broker/pom.xml index 16e0262766f..add83045dcb 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 30b1d2299a5..9f1fd0ad028 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -16,7 +16,33 @@ */ package org.apache.rocketmq.broker; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.stream.Collectors; + import com.google.common.collect.Lists; + import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.acl.plain.PlainAccessValidator; import org.apache.rocketmq.broker.client.ClientHousekeepingService; @@ -34,7 +60,6 @@ import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.latency.BrokerFastFailure; -import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; import org.apache.rocketmq.broker.longpolling.LmqPullRequestHoldService; import org.apache.rocketmq.broker.longpolling.NotifyMessageArrivingListener; import org.apache.rocketmq.broker.longpolling.PullRequestHoldService; @@ -98,6 +123,7 @@ import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.common.stats.MomentStatsItem; import org.apache.rocketmq.common.utils.ServiceProvider; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.Configuration; @@ -126,7 +152,7 @@ import org.apache.rocketmq.store.MessageArrivingListener; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; -import org.apache.rocketmq.store.StoreType; +import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.dledger.DLedgerCommitLog; @@ -141,32 +167,6 @@ import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.timer.TimerMetrics; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.stream.Collectors; - public class BrokerController { protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); @@ -309,7 +309,7 @@ public BrokerController( this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort())); this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()); this.broadcastOffsetManager = new BroadcastOffsetManager(this); - if (isEnableRocksDBStore()) { + if (this.messageStoreConfig.isEnableRocksDBStore()) { this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this); this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this); this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this); @@ -455,10 +455,10 @@ protected void initializeRemotingServer() throws CloneNotSupportedException { * Initialize resources including remoting server and thread executors. */ protected void initializeResources() { - this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, + this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("BrokerControllerScheduledThread", true, getBrokerIdentity())); - this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.sendMessageExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getSendMessageThreadPoolNums(), this.brokerConfig.getSendMessageThreadPoolNums(), 1000 * 60, @@ -466,7 +466,7 @@ protected void initializeResources() { this.sendThreadPoolQueue, new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity())); - this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.pullMessageExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getPullMessageThreadPoolNums(), this.brokerConfig.getPullMessageThreadPoolNums(), 1000 * 60, @@ -474,7 +474,7 @@ protected void initializeResources() { this.pullThreadPoolQueue, new ThreadFactoryImpl("PullMessageThread_", getBrokerIdentity())); - this.litePullMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.litePullMessageExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getLitePullMessageThreadPoolNums(), this.brokerConfig.getLitePullMessageThreadPoolNums(), 1000 * 60, @@ -482,7 +482,7 @@ protected void initializeResources() { this.litePullThreadPoolQueue, new ThreadFactoryImpl("LitePullMessageThread_", getBrokerIdentity())); - this.putMessageFutureExecutor = new BrokerFixedThreadPoolExecutor( + this.putMessageFutureExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getPutMessageFutureThreadPoolNums(), this.brokerConfig.getPutMessageFutureThreadPoolNums(), 1000 * 60, @@ -490,7 +490,7 @@ protected void initializeResources() { this.putThreadPoolQueue, new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity())); - this.ackMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.ackMessageExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getAckMessageThreadPoolNums(), this.brokerConfig.getAckMessageThreadPoolNums(), 1000 * 60, @@ -498,7 +498,7 @@ protected void initializeResources() { this.ackThreadPoolQueue, new ThreadFactoryImpl("AckMessageThread_", getBrokerIdentity())); - this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.queryMessageExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getQueryMessageThreadPoolNums(), this.brokerConfig.getQueryMessageThreadPoolNums(), 1000 * 60, @@ -506,7 +506,7 @@ protected void initializeResources() { this.queryThreadPoolQueue, new ThreadFactoryImpl("QueryMessageThread_", getBrokerIdentity())); - this.adminBrokerExecutor = new BrokerFixedThreadPoolExecutor( + this.adminBrokerExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getAdminBrokerThreadPoolNums(), this.brokerConfig.getAdminBrokerThreadPoolNums(), 1000 * 60, @@ -514,7 +514,7 @@ protected void initializeResources() { this.adminBrokerThreadPoolQueue, new ThreadFactoryImpl("AdminBrokerThread_", getBrokerIdentity())); - this.clientManageExecutor = new BrokerFixedThreadPoolExecutor( + this.clientManageExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getClientManageThreadPoolNums(), this.brokerConfig.getClientManageThreadPoolNums(), 1000 * 60, @@ -522,7 +522,7 @@ protected void initializeResources() { this.clientManagerThreadPoolQueue, new ThreadFactoryImpl("ClientManageThread_", getBrokerIdentity())); - this.heartbeatExecutor = new BrokerFixedThreadPoolExecutor( + this.heartbeatExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getHeartbeatThreadPoolNums(), this.brokerConfig.getHeartbeatThreadPoolNums(), 1000 * 60, @@ -530,7 +530,7 @@ protected void initializeResources() { this.heartbeatThreadPoolQueue, new ThreadFactoryImpl("HeartbeatThread_", true, getBrokerIdentity())); - this.consumerManageExecutor = new BrokerFixedThreadPoolExecutor( + this.consumerManageExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getConsumerManageThreadPoolNums(), this.brokerConfig.getConsumerManageThreadPoolNums(), 1000 * 60, @@ -538,7 +538,7 @@ protected void initializeResources() { this.consumerManagerThreadPoolQueue, new ThreadFactoryImpl("ConsumerManageThread_", true, getBrokerIdentity())); - this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.replyMessageExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getProcessReplyMessageThreadPoolNums(), this.brokerConfig.getProcessReplyMessageThreadPoolNums(), 1000 * 60, @@ -546,7 +546,7 @@ protected void initializeResources() { this.replyThreadPoolQueue, new ThreadFactoryImpl("ProcessReplyMessageThread_", getBrokerIdentity())); - this.endTransactionExecutor = new BrokerFixedThreadPoolExecutor( + this.endTransactionExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getEndTransactionThreadPoolNums(), this.brokerConfig.getEndTransactionThreadPoolNums(), 1000 * 60, @@ -554,7 +554,7 @@ protected void initializeResources() { this.endTransactionThreadPoolQueue, new ThreadFactoryImpl("EndTransactionThread_", getBrokerIdentity())); - this.loadBalanceExecutor = new BrokerFixedThreadPoolExecutor( + this.loadBalanceExecutor = ThreadUtils.newThreadPoolExecutor( this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), 1000 * 60, @@ -562,9 +562,9 @@ protected void initializeResources() { this.loadBalanceThreadPoolQueue, new ThreadFactoryImpl("LoadBalanceProcessorThread_", getBrokerIdentity())); - this.syncBrokerMemberGroupExecutorService = new ScheduledThreadPoolExecutor(1, + this.syncBrokerMemberGroupExecutorService = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("BrokerControllerSyncBrokerScheduledThread", getBrokerIdentity())); - this.brokerHeartbeatExecutorService = new ScheduledThreadPoolExecutor(1, + this.brokerHeartbeatExecutorService = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("BrokerControllerHeartbeatScheduledThread", getBrokerIdentity())); this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this); @@ -663,7 +663,7 @@ public void run() { BrokerController.this.getSlaveSynchronize().syncAll(); lastSyncTimeMs = System.currentTimeMillis(); } - + //timer checkpoint, latency-sensitive, so sync it more frequently if (messageStoreConfig.isTimerWheelEnable()) { BrokerController.this.getSlaveSynchronize().syncTimerCheckPoint(); @@ -698,17 +698,6 @@ protected void initializeScheduledTasks() { initializeBrokerScheduledTasks(); - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - try { - BrokerController.this.brokerOuterAPI.refreshMetadata(); - } catch (Exception e) { - LOG.error("ScheduledTask refresh metadata exception", e); - } - } - }, 10, 5, TimeUnit.SECONDS); - if (this.brokerConfig.getNamesrvAddr() != null) { this.updateNamesrvAddr(); LOG.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr()); @@ -759,7 +748,12 @@ public boolean initializeMetadata() { public boolean initializeMessageStore() { boolean result = true; try { - DefaultMessageStore defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); + DefaultMessageStore defaultMessageStore; + if (this.messageStoreConfig.isEnableRocksDBStore()) { + defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); + } else { + defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); + } if (messageStoreConfig.isEnableDLegerCommitLog()) { DLedgerRoleChangeHandler roleChangeHandler = @@ -956,16 +950,16 @@ private void initialTransaction() { this.transactionalMessageService = ServiceProvider.loadClass(TransactionalMessageService.class); if (null == this.transactionalMessageService) { this.transactionalMessageService = new TransactionalMessageServiceImpl( - new TransactionalMessageBridge(this, this.getMessageStore())); + new TransactionalMessageBridge(this, this.getMessageStore())); LOG.warn("Load default transaction message hook service: {}", - TransactionalMessageServiceImpl.class.getSimpleName()); + TransactionalMessageServiceImpl.class.getSimpleName()); } this.transactionalMessageCheckListener = ServiceProvider.loadClass( - AbstractTransactionalMessageCheckListener.class); + AbstractTransactionalMessageCheckListener.class); if (null == this.transactionalMessageCheckListener) { this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener(); LOG.warn("Load default discard message hook service: {}", - DefaultTransactionalMessageCheckListener.class.getSimpleName()); + DefaultTransactionalMessageCheckListener.class.getSimpleName()); } this.transactionalMessageCheckListener.setBrokerController(this); this.transactionalMessageCheckService = new TransactionalMessageCheckService(this); @@ -1194,6 +1188,7 @@ public void printWaterMark() { LOG_WATER_MARK.info("[WATERMARK] ClientManager Queue Size: {} SlowTimeMills: {}", this.clientManagerThreadPoolQueue.size(), this.headSlowTimeMills(this.clientManagerThreadPoolQueue)); LOG_WATER_MARK.info("[WATERMARK] Heartbeat Queue Size: {} SlowTimeMills: {}", this.heartbeatThreadPoolQueue.size(), this.headSlowTimeMills(this.heartbeatThreadPoolQueue)); LOG_WATER_MARK.info("[WATERMARK] Ack Queue Size: {} SlowTimeMills: {}", this.ackThreadPoolQueue.size(), headSlowTimeMills(this.ackThreadPoolQueue)); + LOG_WATER_MARK.info("[WATERMARK] Admin Queue Size: {} SlowTimeMills: {}", this.adminBrokerThreadPoolQueue.size(), headSlowTimeMills(this.adminBrokerThreadPoolQueue)); } public MessageStore getMessageStore() { @@ -1313,6 +1308,10 @@ protected void shutdownBasicService() { this.fastRemotingServer.shutdown(); } + if (this.brokerMetricsManager != null) { + this.brokerMetricsManager.shutdown(); + } + if (this.brokerStatsManager != null) { this.brokerStatsManager.shutdown(); } @@ -1335,6 +1334,10 @@ protected void shutdownBasicService() { this.ackMessageProcessor.shutdownPopReviveService(); } + if (this.transactionalMessageService != null) { + this.transactionalMessageService.close(); + } + if (this.notificationProcessor != null) { this.notificationProcessor.getPopLongPollingService().shutdown(); } @@ -1682,6 +1685,17 @@ public void run0() { if (brokerConfig.isSkipPreOnline()) { startServiceWithoutCondition(); } + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.brokerOuterAPI.refreshMetadata(); + } catch (Exception e) { + LOG.error("ScheduledTask refresh metadata exception", e); + } + } + }, 10, 5, TimeUnit.SECONDS); } protected void scheduleSendHeartbeat() { @@ -1733,7 +1747,8 @@ public synchronized void registerIncrementBrokerData(List topicConf new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), - this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); + topicConfig.getPerm() + & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); } else { registerTopicConfig = new TopicConfig(topicConfig); } @@ -1757,29 +1772,34 @@ public synchronized void registerIncrementBrokerData(List topicConf } public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) { + ConcurrentMap topicConfigMap = this.getTopicConfigManager().getTopicConfigTable(); + ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); - TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); - - topicConfigWrapper.setDataVersion(this.getTopicConfigManager().getDataVersion()); - topicConfigWrapper.setTopicConfigTable(this.getTopicConfigManager().getTopicConfigTable()); - - topicConfigWrapper.setTopicQueueMappingInfoMap(this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream().map( - entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue())) - ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - - if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) - || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { - ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); - for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) { - TopicConfig tmp = + for (TopicConfig topicConfig : topicConfigMap.values()) { + if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) + || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { + topicConfigTable.put(topicConfig.getTopicName(), new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), - topicConfig.getPerm() & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); - topicConfigTable.put(topicConfig.getTopicName(), tmp); + topicConfig.getPerm() & getBrokerConfig().getBrokerPermission())); + } else { + topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + + if (this.brokerConfig.isEnableSplitRegistration() + && topicConfigTable.size() >= this.brokerConfig.getSplitRegistrationSize()) { + TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildSerializeWrapper(topicConfigTable); + doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper); + topicConfigTable.clear(); } - topicConfigWrapper.setTopicConfigTable(topicConfigTable); } - if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), + Map topicQueueMappingInfoMap = this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream() + .map(entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager(). + buildSerializeWrapper(topicConfigTable, topicQueueMappingInfoMap); + if (this.brokerConfig.isEnableSplitRegistration() || forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), this.getBrokerAddr(), this.brokerConfig.getBrokerName(), this.brokerConfig.getBrokerId(), @@ -1793,7 +1813,7 @@ protected void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway, TopicConfigSerializeWrapper topicConfigWrapper) { if (shutdown) { - BrokerController.LOG.info("BrokerController#doResterBrokerAll: broker has shutdown, no need to register any more."); + BrokerController.LOG.info("BrokerController#doRegisterBrokerAll: broker has shutdown, no need to register any more."); return; } List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( @@ -2398,8 +2418,4 @@ public ColdDataCgCtrService getColdDataCgCtrService() { public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) { this.coldDataCgCtrService = coldDataCgCtrService; } - - public boolean isEnableRocksDBStore() { - return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.messageStoreConfig.getStoreType()); - } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java index 98e5f450f3f..cbb81f632b4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java @@ -18,11 +18,11 @@ import io.netty.channel.Channel; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; @@ -35,7 +35,7 @@ public class ClientHousekeepingService implements ChannelEventListener { public ClientHousekeepingService(final BrokerController brokerController) { this.brokerController = brokerController; - scheduledExecutorService = new ScheduledThreadPoolExecutor(1, + scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("ClientHousekeepingScheduledThread", brokerController.getBrokerIdentity())); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java index 2ce036a0ffc..d17a2a5470c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.AbstractBrokerRunnable; @@ -37,7 +36,7 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen private final BrokerController brokerController; private final int cacheSize = 8096; - private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, + private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ThreadUtils.newGenericThreadFactory("DefaultConsumerIdsChangeListener", true)); private ConcurrentHashMap> consumerChannelMap = new ConcurrentHashMap<>(cacheSize); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java index abae7cdb01a..a1d711cb275 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java @@ -27,10 +27,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; @@ -42,6 +40,7 @@ import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.protocol.EpochEntry; @@ -107,9 +106,9 @@ public class ReplicasManager { public ReplicasManager(final BrokerController brokerController) { this.brokerController = brokerController; this.brokerOuterAPI = brokerController.getBrokerOuterAPI(); - this.scheduledService = Executors.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity())); - this.executorService = Executors.newFixedThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity())); - this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, + this.scheduledService = ThreadUtils.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity())); + this.executorService = ThreadUtils.newThreadPoolExecutor(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity())); + this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("ReplicasManager_scan_thread_", brokerController.getBrokerIdentity())); this.haService = (AutoSwitchHAService) brokerController.getMessageStore().getHaService(); this.brokerConfig = brokerController.getBrokerConfig(); @@ -225,7 +224,7 @@ public void shutdown() { public synchronized void changeBrokerRole(final Long newMasterBrokerId, final String newMasterAddress, final Integer newMasterEpoch, - final Integer syncStateSetEpoch, final Set syncStateSet) { + final Integer syncStateSetEpoch, final Set syncStateSet) throws Exception { if (newMasterBrokerId != null && newMasterEpoch > this.masterEpoch) { if (newMasterBrokerId.equals(this.brokerControllerId)) { changeToMaster(newMasterEpoch, syncStateSetEpoch, syncStateSet); @@ -235,7 +234,7 @@ public synchronized void changeBrokerRole(final Long newMasterBrokerId, final St } } - public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set syncStateSet) { + public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set syncStateSet) throws Exception { synchronized (this) { if (newMasterEpoch > this.masterEpoch) { LOGGER.info("Begin to change to master, brokerName:{}, replicas:{}, new Epoch:{}", this.brokerConfig.getBrokerName(), this.brokerAddress, newMasterEpoch); @@ -542,7 +541,7 @@ private boolean createMetadataFileAndDeleteTemp() { this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId()); this.tempBrokerMetadata.clear(); this.brokerControllerId = this.brokerMetadata.getBrokerId(); - this.haService.setBrokerControllerId(this.brokerControllerId); + this.haService.setLocalBrokerId(this.brokerControllerId); return true; } catch (Exception e) { LOGGER.error("fail to create metadata file", e); @@ -594,7 +593,7 @@ private void confirmNowRegisteringState() { if (this.brokerMetadata.isLoaded()) { this.registerState = RegisterState.CREATE_METADATA_FILE_DONE; this.brokerControllerId = brokerMetadata.getBrokerId(); - this.haService.setBrokerControllerId(this.brokerControllerId); + this.haService.setLocalBrokerId(this.brokerControllerId); return; } // 2. check if temp metadata exist @@ -735,23 +734,26 @@ private void schedulingCheckSyncStateSet() { if (this.checkSyncStateSetTaskFuture != null) { this.checkSyncStateSetTaskFuture.cancel(false); } - this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> { - checkSyncStateSetAndDoReport(); - }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); + this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000, + this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); } private void checkSyncStateSetAndDoReport() { - final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); - newSyncStateSet.add(this.brokerControllerId); - synchronized (this) { - if (this.syncStateSet != null) { - // Check if syncStateSet changed - if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { - return; + try { + final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); + newSyncStateSet.add(this.brokerControllerId); + synchronized (this) { + if (this.syncStateSet != null) { + // Check if syncStateSet changed + if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { + return; + } } } + doReportSyncStateSetChanged(newSyncStateSet); + } catch (Exception e) { + LOGGER.error("Check syncStateSet error", e); } - doReportSyncStateSetChanged(newSyncStateSet); } private void doReportSyncStateSetChanged(Set newSyncStateSet) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java index 75023ee1b8b..e6cb97640bd 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java @@ -21,12 +21,12 @@ import io.openmessaging.storage.dledger.MemberState; import io.openmessaging.storage.dledger.utils.DLedgerUtils; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; @@ -49,7 +49,7 @@ public DLedgerRoleChangeHandler(BrokerController brokerController, DefaultMessag this.messageStore = messageStore; this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); this.dLegerServer = dLedgerCommitLog.getdLedgerServer(); - this.executorService = Executors.newSingleThreadExecutor( + this.executorService = ThreadUtils.newSingleThreadExecutor( new ThreadFactoryImpl("DLegerRoleChangeHandler_", brokerController.getBrokerIdentity())); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java index 7c350fc1d7d..6a081748014 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java @@ -24,7 +24,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.broker.BrokerController; @@ -43,6 +42,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -72,7 +72,7 @@ public EscapeBridge(BrokerController brokerController) { public void start() throws Exception { if (brokerController.getBrokerConfig().isEnableSlaveActingMaster() && brokerController.getBrokerConfig().isEnableRemoteEscape()) { final BlockingQueue asyncSenderThreadPoolQueue = new LinkedBlockingQueue<>(50000); - this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( + this.defaultAsyncSenderExecutor = ThreadUtils.newThreadPoolExecutor( Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors(), 1000 * 60, diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java index d3d0bc8ba3a..3b6e9dc676e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java @@ -18,13 +18,14 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.AbstractBrokerRunnable; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.future.FutureTaskExt; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.netty.RequestTask; @@ -43,7 +44,7 @@ public class BrokerFastFailure { public BrokerFastFailure(final BrokerController brokerController) { this.brokerController = brokerController; - this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, + this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true, brokerController == null ? null : brokerController.getBrokerConfig())); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java deleted file mode 100644 index d2d1143a348..00000000000 --- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.broker.latency; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.RejectedExecutionHandler; -import java.util.concurrent.RunnableFuture; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -public class BrokerFixedThreadPoolExecutor extends ThreadPoolExecutor { - public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, - final TimeUnit unit, - final BlockingQueue workQueue) { - super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); - } - - public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, - final TimeUnit unit, - final BlockingQueue workQueue, final ThreadFactory threadFactory) { - super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); - } - - public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, - final TimeUnit unit, - final BlockingQueue workQueue, final RejectedExecutionHandler handler) { - super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler); - } - - public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, - final TimeUnit unit, - final BlockingQueue workQueue, final ThreadFactory threadFactory, - final RejectedExecutionHandler handler) { - super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); - } - - @Override - protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { - return new FutureTaskExt<>(runnable, value); - } -} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java index 6af5afc1413..39af18b9faa 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java @@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongHistogram; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.ObservableLongGauge; -import io.opentelemetry.exporter.logging.LoggingMetricExporter; +import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; @@ -36,6 +36,7 @@ import io.opentelemetry.sdk.metrics.View; import io.opentelemetry.sdk.metrics.ViewBuilder; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.export.MetricExporter; import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; import io.opentelemetry.sdk.resources.Resource; @@ -113,7 +114,7 @@ public class BrokerMetricsManager { private OtlpGrpcMetricExporter metricExporter; private PeriodicMetricReader periodicMetricReader; private PrometheusHttpServer prometheusHttpServer; - private LoggingMetricExporter loggingMetricExporter; + private MetricExporter loggingMetricExporter; private Meter brokerMeter; public static Supplier attributesBuilderSupplier = Attributes::builder; @@ -327,8 +328,8 @@ private void init() { if (metricsExporterType == MetricsExporterType.LOG) { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); - loggingMetricExporter = LoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); - java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); + java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) .setInterval(brokerConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) .build(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java index 5695a335624..05b53b0bcf2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java @@ -33,7 +33,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { public RocksDBConsumerOffsetManager(BrokerController brokerController) { super(brokerController); - this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); + this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override @@ -49,7 +49,7 @@ public boolean stop() { @Override protected void removeConsumerOffset(String topicAtGroup) { try { - byte[] keyBytes = topicAtGroup.getBytes(DataConverter.charset); + byte[] keyBytes = topicAtGroup.getBytes(DataConverter.CHARSET_UTF8); this.rocksDBConfigManager.delete(keyBytes); } catch (Exception e) { LOG.error("kv remove consumerOffset Failed, {}", topicAtGroup); @@ -58,7 +58,7 @@ protected void removeConsumerOffset(String topicAtGroup) { @Override protected void decode0(final byte[] key, final byte[] body) { - String topicAtGroup = new String(key, DataConverter.charset); + String topicAtGroup = new String(key, DataConverter.CHARSET_UTF8); RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class); this.offsetTable.put(topicAtGroup, wrapper.getOffsetTable()); @@ -93,7 +93,7 @@ public synchronized void persist() { } private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupName, final ConcurrentMap offsetMap) throws Exception { - byte[] keyBytes = topicGroupName.getBytes(DataConverter.charset); + byte[] keyBytes = topicGroupName.getBytes(DataConverter.CHARSET_UTF8); RocksDBOffsetSerializeWrapper wrapper = new RocksDBOffsetSerializeWrapper(); wrapper.setOffsetTable(offsetMap); byte[] valueBytes = JSON.toJSONBytes(wrapper, SerializerFeature.BrowserCompatible); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index ae81e8b11df..6fde48dd995 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -27,9 +27,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.exception.MQBrokerException; @@ -59,6 +59,7 @@ import org.apache.rocketmq.common.namesrv.TopAddressing; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.InvokeCallback; @@ -72,6 +73,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo; import org.apache.rocketmq.remoting.protocol.DataVersion; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -106,6 +108,8 @@ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader; @@ -123,8 +127,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult; import org.apache.rocketmq.remoting.protocol.route.BrokerData; @@ -144,13 +146,12 @@ public class BrokerOuterAPI { private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final RemotingClient remotingClient; private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr()); - private final BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, + private final ExecutorService brokerOuterExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true)); private final ClientMetadata clientMetadata; private final RpcClient rpcClient; private String nameSrvAddr = null; - public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { this(nettyClientConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata()); } @@ -458,7 +459,7 @@ public List registerBrokerAll( * @param filterServerList * @param oneway * @param timeoutMills - * @param compressed default false + * @param compressed default false * @return */ public List registerBrokerAll( @@ -642,7 +643,6 @@ public void registerSingleTopicAll( queueDatas.add(queueData); final byte[] topicRouteBody = topicRouteData.encode(); - List nameServerAddressList = this.remotingClient.getNameServerAddressList(); final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size()); for (final String namesrvAddr : nameServerAddressList) { @@ -909,25 +909,33 @@ public void lockBatchMQAsync( RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); request.setBody(requestBody.encode()); - this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { - if (callback == null) { - return; + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + } - try { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (response.getCode() == ResponseCode.SUCCESS) { - LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), - LockBatchResponseBody.class); - Set messageQueues = responseBody.getLockOKMQSet(); - callback.onSuccess(messageQueues); - } else { - callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); - } + @Override + public void operationSucceed(RemotingCommand response) { + if (callback == null) { + return; } - } catch (Throwable ignored) { + if (response.getCode() == ResponseCode.SUCCESS) { + LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), + LockBatchResponseBody.class); + Set messageQueues = responseBody.getLockOKMQSet(); + callback.onSuccess(messageQueues); + } else { + callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); + } + } + @Override + public void operationFail(Throwable throwable) { + if (callback == null) { + return; + } + callback.onException(throwable); } }); } @@ -941,22 +949,30 @@ public void unlockBatchMQAsync( request.setBody(requestBody.encode()); - this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { - if (callback == null) { - return; + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + } - try { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (response.getCode() == ResponseCode.SUCCESS) { - callback.onSuccess(); - } else { - callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); - } + @Override + public void operationSucceed(RemotingCommand response) { + if (callback == null) { + return; } - } catch (Throwable ignored) { + if (response.getCode() == ResponseCode.SUCCESS) { + callback.onSuccess(); + } else { + callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); + } + } + @Override + public void operationFail(Throwable throwable) { + if (callback == null) { + return; + } + callback.onException(throwable); } }); } @@ -982,21 +998,27 @@ public CompletableFuture sendMessageToSpecificBrokerAsync(String bro CompletableFuture cf = new CompletableFuture<>(); final String msgId = msg.getMsgId(); try { - this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (null != response) { - SendResult sendResult = null; + this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { try { - sendResult = this.processSendResponse(brokerName, msg, response); + SendResult sendResult = processSendResponse(brokerName, msg, response); cf.complete(sendResult); } catch (MQBrokerException | RemotingCommandException e) { LOGGER.error("processSendResponse in sendMessageToSpecificBrokerAsync failed, msgId=" + msgId, e); cf.completeExceptionally(e); } - } else { - cf.complete(null); } + @Override + public void operationFail(Throwable throwable) { + cf.completeExceptionally(throwable); + } }); } catch (Throwable t) { LOGGER.error("invokeAsync failed in sendMessageToSpecificBrokerAsync, msgId=" + msgId, t); @@ -1056,7 +1078,7 @@ private SendResult processSendResponse( } if (sendStatus != null) { SendMessageResponseHeader responseHeader = - (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); + (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); //If namespace not null , reset Topic without namespace. String topic = msg.getTopic(); @@ -1072,8 +1094,8 @@ private SendResult processSendResponse( uniqMsgId = sb.toString(); } SendResult sendResult = new SendResult(sendStatus, - uniqMsgId, - responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); + uniqMsgId, + responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); sendResult.setTransactionId(responseHeader.getTransactionId()); String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH); @@ -1092,7 +1114,7 @@ private SendResult processSendResponse( throw new MQBrokerException(response.getCode(), response.getRemark()); } - public BrokerFixedThreadPoolExecutor getBrokerOuterExecutor() { + public ExecutorService getBrokerOuterExecutor() { return brokerOuterExecutor; } @@ -1217,8 +1239,9 @@ public SyncStateSet alterSyncStateSet( /** * Broker try to elect itself as a master in broker set */ - public Pair> brokerElect(String controllerAddress, String clusterName, String brokerName, - Long brokerId) throws Exception { + public Pair> brokerElect(String controllerAddress, String clusterName, + String brokerName, + Long brokerId) throws Exception { final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader); @@ -1236,7 +1259,8 @@ public Pair> brokerElect(String controllerA throw new MQBrokerException(response.getCode(), response.getRemark()); } - public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, final String controllerAddress) throws Exception { + public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, + final String controllerAddress) throws Exception { final GetNextBrokerIdRequestHeader requestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName); final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, requestHeader); final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); @@ -1247,7 +1271,8 @@ public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, f throw new MQBrokerException(response.getCode(), response.getRemark()); } - public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { + public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, + final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { final ApplyBrokerIdRequestHeader requestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode); final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, requestHeader); final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); @@ -1258,7 +1283,9 @@ public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final throw new MQBrokerException(response.getCode(), response.getRemark()); } - public Pair> registerBrokerToController(final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, final String controllerAddress) throws Exception { + public Pair> registerBrokerToController( + final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, + final String controllerAddress) throws Exception { final RegisterBrokerToControllerRequestHeader requestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress); final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, requestHeader); final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); @@ -1354,16 +1381,25 @@ public CompletableFuture pullMessageFromSpecificBrokerAsync(String b RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); CompletableFuture pullResultFuture = new CompletableFuture<>(); - this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - if (responseFuture.getCause() != null) { - pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); - return; + this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + } - try { - PullResultExt pullResultExt = this.processPullResponse(responseFuture.getResponseCommand(), brokerAddr); - this.processPullResult(pullResultExt, brokerName, queueId); - pullResultFuture.complete(pullResultExt); - } catch (Exception e) { + + @Override + public void operationSucceed(RemotingCommand response) { + try { + PullResultExt pullResultExt = processPullResponse(response, brokerAddr); + processPullResult(pullResultExt, brokerName, queueId); + pullResultFuture.complete(pullResultExt); + } catch (Exception e) { + pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); + } + } + + @Override + public void operationFail(Throwable throwable) { pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); } }); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java index 687811409e0..59a3e63b2ab 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java @@ -19,6 +19,7 @@ import com.alibaba.fastjson.JSON; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import java.util.BitSet; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.metrics.PopMetricsManager; import org.apache.rocketmq.common.KeyBuilder; @@ -186,46 +187,7 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo); if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { - // order - String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; - long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); - if (ackOffset < oldOffset) { - return; - } - while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { - } - try { - oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); - if (ackOffset < oldOffset) { - return; - } - long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( - topic, consumeGroup, - qId, ackOffset, - popTime); - if (nextOffset > -1) { - if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( - topic, consumeGroup, qId)) { - this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), - consumeGroup, topic, qId, nextOffset); - } - if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, - consumeGroup, qId, invisibleTime)) { - this.brokerController.getPopMessageProcessor().notifyMessageArriving( - topic, consumeGroup, qId); - } - } else if (nextOffset == -1) { - String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", - lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress()); - POP_LOGGER.warn(errorInfo); - response.setCode(ResponseCode.MESSAGE_ILLEGAL); - response.setRemark(errorInfo); - return; - } - } finally { - this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); - } - brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); + ackOrderly(topic, consumeGroup, qId, ackOffset, popTime, invisibleTime, channel, response); return; } @@ -250,17 +212,22 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA } BatchAckMsg batchAckMsg = new BatchAckMsg(); - for (int i = 0; batchAck.getBitSet() != null && i < batchAck.getBitSet().length(); i++) { - if (!batchAck.getBitSet().get(i)) { - continue; + BitSet bitSet = batchAck.getBitSet(); + for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { + if (i == Integer.MAX_VALUE) { + break; } long offset = startOffset + i; if (offset < minOffset || offset > maxOffset) { continue; } - batchAckMsg.getAckOffsetList().add(offset); + if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { + ackOrderly(topic, consumeGroup, qId, offset, popTime, invisibleTime, channel, response); + } else { + batchAckMsg.getAckOffsetList().add(offset); + } } - if (batchAckMsg.getAckOffsetList().isEmpty()) { + if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE || batchAckMsg.getAckOffsetList().isEmpty()) { return; } @@ -286,7 +253,7 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(rqId); if (ackMsg instanceof BatchAckMsg) { msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); @@ -311,4 +278,46 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus()); brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); } + + protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, long invisibleTime, Channel channel, RemotingCommand response) { + String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; + long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); + if (ackOffset < oldOffset) { + return; + } + while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { + } + try { + oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); + if (ackOffset < oldOffset) { + return; + } + long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( + topic, consumeGroup, + qId, ackOffset, + popTime); + if (nextOffset > -1) { + if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( + topic, consumeGroup, qId)) { + this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), + consumeGroup, topic, qId, nextOffset); + } + if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, + consumeGroup, qId, invisibleTime)) { + this.brokerController.getPopMessageProcessor().notifyMessageArriving( + topic, consumeGroup, qId); + } + } else if (nextOffset == -1) { + String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", + lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress()); + POP_LOGGER.warn(errorInfo); + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark(errorInfo); + return; + } + } finally { + this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); + } + brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, 1); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index bbddcec2d7f..dd4ec960fe5 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -51,6 +51,7 @@ import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.KeyBuilder; import org.apache.rocketmq.common.LockCallback; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; @@ -405,9 +406,6 @@ public boolean rejectRequest() { private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - if (validateSlave(response)) { - return response; - } final CreateTopicRequestHeader requestHeader = (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); @@ -518,9 +516,6 @@ private synchronized RemotingCommand updateAndCreateStaticTopic(ChannelHandlerCo private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - if (validateSlave(response)) { - return response; - } DeleteTopicRequestHeader requestHeader = (DeleteTopicRequestHeader) request.decodeCommandCustomHeader(DeleteTopicRequestHeader.class); @@ -542,16 +537,33 @@ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, } } - this.brokerController.getTopicConfigManager().deleteTopicConfig(requestHeader.getTopic()); - this.brokerController.getTopicQueueMappingManager().delete(requestHeader.getTopic()); - this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(requestHeader.getTopic()); - this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(requestHeader.getTopic()); - this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(requestHeader.getTopic())); + final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); + // delete pop retry topics first + try { + for (String group : groups) { + final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); + if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { + deleteTopicInBroker(popRetryTopic); + } + } + // delete topic + deleteTopicInBroker(topic); + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; } + private void deleteTopicInBroker(String topic) { + this.brokerController.getTopicConfigManager().deleteTopicConfig(topic); + this.brokerController.getTopicQueueMappingManager().delete(topic); + this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(topic); + this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(topic); + this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic)); + } + private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); @@ -1399,9 +1411,6 @@ public void onException(Throwable e) { private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - if (validateSlave(response)) { - return response; - } LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); @@ -1466,9 +1475,6 @@ private RemotingCommand getAllSubscriptionGroup(ChannelHandlerContext ctx, private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - if (validateSlave(response)) { - return response; - } DeleteSubscriptionGroupRequestHeader requestHeader = (DeleteSubscriptionGroupRequestHeader) request.decodeCommandCustomHeader(DeleteSubscriptionGroupRequestHeader.class); @@ -2079,7 +2085,11 @@ private RemotingCommand getSystemTopicListFromBroker(ChannelHandlerContext ctx, public RemotingCommand cleanExpiredConsumeQueue() { LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: start."); final RemotingCommand response = RemotingCommand.createResponseCommand(null); - brokerController.getMessageStore().cleanExpiredConsumerQueue(); + try { + brokerController.getMessageStore().cleanExpiredConsumerQueue(); + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: end."); response.setCode(ResponseCode.SUCCESS); response.setRemark(null); @@ -2734,10 +2744,16 @@ private RemotingCommand getBrokerEpochCache(ChannelHandlerContext ctx, RemotingC final ReplicasManager replicasManager = this.brokerController.getReplicasManager(); assert replicasManager != null; final BrokerConfig brokerConfig = this.brokerController.getBrokerConfig(); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + if (!brokerConfig.isEnableControllerMode()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("this request only for controllerMode "); + return response; + } final EpochEntryCache entryCache = new EpochEntryCache(brokerConfig.getBrokerClusterName(), - brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset()); + brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset()); - final RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setBody(entryCache.encode()); response.setCode(ResponseCode.SUCCESS); response.setRemark(null); @@ -2773,7 +2789,11 @@ private RemotingCommand notifyBrokerRoleChanged(ChannelHandlerContext ctx, final ReplicasManager replicasManager = this.brokerController.getReplicasManager(); if (replicasManager != null) { - replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet()); + try { + replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet()); + } catch (Exception e) { + throw new RemotingCommandException(e.getMessage()); + } } response.setCode(ResponseCode.SUCCESS); response.setRemark(null); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java index 2ccdf07f6aa..bdfffff096a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java @@ -180,7 +180,7 @@ private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, Str } msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(rqId); msgInner.setTags(PopAckConstants.ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); @@ -216,7 +216,7 @@ private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader ck.addDiff(0); ck.setBrokerName(brokerName); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java index a8358c4ffb0..e1e0e13e53b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java @@ -129,8 +129,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re } int randomQ = random.nextInt(100); int reviveQid = randomQ % this.brokerController.getBrokerConfig().getReviveQueueNum(); - int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg(); - GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg); + GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums()); boolean needRetry = randomQ % 5 == 0; long popTime = System.currentTimeMillis(); long restNum = 0; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java index b7ba8ad4a20..8a85dd8fec8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java @@ -633,7 +633,7 @@ private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgI ackMsg.setQueueId(point.getQueueId()); ackMsg.setPopTime(point.getPopTime()); msgInner.setTopic(popMessageProcessor.reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(pointWrapper.getReviveQueueId()); msgInner.setTags(PopAckConstants.ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); @@ -673,7 +673,7 @@ private boolean putBatchAckToStore(final PopCheckPointWrapper pointWrapper, fina batchAckMsg.setQueueId(point.getQueueId()); batchAckMsg.setPopTime(point.getPopTime()); msgInner.setTopic(popMessageProcessor.reviveTopic); - msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(pointWrapper.getReviveQueueId()); msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 441f7de08a1..f5d07c5aae9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -347,8 +347,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() % this.brokerController.getBrokerConfig().getReviveQueueNum()); } - int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg(); - GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg); + GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums()); ExpressionMessageFilter finalMessageFilter = messageFilter; StringBuilder finalOrderCountInfo = orderCountInfo; @@ -686,7 +685,7 @@ public final MessageExtBrokerInner buildCkMsg(final PopCheckPoint ck, final int MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index 93167db373a..3fb689ed6a9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -129,7 +129,7 @@ private boolean reviveRetry(PopCheckPoint popCheckPoint, MessageExt messageExt) PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); PopMetricsManager.incPopReviveRetryMessageCount(popCheckPoint, putMessageResult.getPutMessageStatus()); if (brokerController.getBrokerConfig().isEnablePopLog()) { - POP_LOGGER.info("reviveQueueId={},retry msg , ck={}, msg queueId {}, offset {}, reviveDelay={}, result is {} ", + POP_LOGGER.info("reviveQueueId={},retry msg, ck={}, msg queueId {}, offset {}, reviveDelay={}, result is {} ", queueId, popCheckPoint, messageExt.getQueueId(), messageExt.getQueueOffset(), (System.currentTimeMillis() - popCheckPoint.getReviveTime()) / 1000, putMessageResult); } @@ -319,7 +319,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { // offset self amend while (true) { if (!shouldRunPopRevive) { - POP_LOGGER.info("slave skip scan , revive topic={}, reviveQueueId={}", reviveTopic, queueId); + POP_LOGGER.info("slave skip scan, revive topic={}, reviveQueueId={}", reviveTopic, queueId); break; } List messageExts = getReviveMessage(offset, queueId); @@ -351,12 +351,12 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { noMsgCount = 0; } if (System.currentTimeMillis() - startScanTime > brokerController.getBrokerConfig().getReviveScanTime()) { - POP_LOGGER.info("reviveQueueId={}, scan timeout ", queueId); + POP_LOGGER.info("reviveQueueId={}, scan timeout ", queueId); break; } for (MessageExt messageExt : messageExts) { if (PopAckConstants.CK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.charset); + String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={},find ck, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } @@ -371,9 +371,9 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { firstRt = point.getReviveTime(); } } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.charset); + String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); if (brokerController.getBrokerConfig().isEnablePopLog()) { - POP_LOGGER.info("reviveQueueId={},find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + POP_LOGGER.info("reviveQueueId={}, find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class); PopMetricsManager.incPopReviveAckGetCount(ackMsg, queueId); @@ -395,7 +395,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { } } } else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.charset); + String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } @@ -465,15 +465,15 @@ private PopCheckPoint createMockCkForAck(AckMsg ackMsg, long reviveOffset) { protected void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwable { ArrayList sortList = consumeReviveObj.genSortList(); - POP_LOGGER.info("reviveQueueId={},ck listSize={}", queueId, sortList.size()); + POP_LOGGER.info("reviveQueueId={}, ck listSize={}", queueId, sortList.size()); if (sortList.size() != 0) { - POP_LOGGER.info("reviveQueueId={}, 1st ck, startOffset={}, reviveOffset={} ; last ck, startOffset={}, reviveOffset={}", queueId, sortList.get(0).getStartOffset(), + POP_LOGGER.info("reviveQueueId={}, 1st ck, startOffset={}, reviveOffset={}; last ck, startOffset={}, reviveOffset={}", queueId, sortList.get(0).getStartOffset(), sortList.get(0).getReviveOffset(), sortList.get(sortList.size() - 1).getStartOffset(), sortList.get(sortList.size() - 1).getReviveOffset()); } long newOffset = consumeReviveObj.oldOffset; for (PopCheckPoint popCheckPoint : sortList) { if (!shouldRunPopRevive) { - POP_LOGGER.info("slave skip ck process , revive topic={}, reviveQueueId={}", reviveTopic, queueId); + POP_LOGGER.info("slave skip ck process, revive topic={}, reviveQueueId={}", reviveTopic, queueId); break; } if (consumeReviveObj.endTime - popCheckPoint.getReviveTime() <= (PopAckConstants.ackTimeInterval + PopAckConstants.SECOND)) { @@ -483,12 +483,12 @@ protected void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwabl // check normal topic, skip ck , if normal topic is not exist String normalTopic = KeyBuilder.parseNormalTopic(popCheckPoint.getTopic(), popCheckPoint.getCId()); if (brokerController.getTopicConfigManager().selectTopicConfig(normalTopic) == null) { - POP_LOGGER.warn("reviveQueueId={},can not get normal topic {} , then continue ", queueId, popCheckPoint.getTopic()); + POP_LOGGER.warn("reviveQueueId={}, can not get normal topic {}, then continue", queueId, popCheckPoint.getTopic()); newOffset = popCheckPoint.getReviveOffset(); continue; } if (null == brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(popCheckPoint.getCId())) { - POP_LOGGER.warn("reviveQueueId={},can not get cid {} , then continue ", queueId, popCheckPoint.getCId()); + POP_LOGGER.warn("reviveQueueId={}, can not get cid {}, then continue", queueId, popCheckPoint.getCId()); newOffset = popCheckPoint.getReviveOffset(); continue; } @@ -520,7 +520,7 @@ protected void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwabl private void reviveMsgFromCk(PopCheckPoint popCheckPoint) { if (!shouldRunPopRevive) { - POP_LOGGER.info("slave skip retry , revive topic={}, reviveQueueId={}", reviveTopic, queueId); + POP_LOGGER.info("slave skip retry, revive topic={}, reviveQueueId={}", reviveTopic, queueId); return; } inflightReviveRequestMap.put(popCheckPoint, new Pair<>(System.currentTimeMillis(), false)); @@ -595,6 +595,7 @@ private void rePutCK(PopCheckPoint oldCK, Pair pair) { newCk.setCId(oldCK.getCId()); newCk.setTopic(oldCK.getTopic()); newCk.setQueueId(oldCK.getQueueId()); + newCk.setBrokerName(oldCK.getBrokerName()); newCk.addDiff(0); MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId); brokerController.getMessageStore().putMessage(ckMsg); @@ -645,7 +646,7 @@ public void run() { consumeReviveMessage(consumeReviveObj); if (!shouldRunPopRevive) { - POP_LOGGER.info("slave skip scan , revive topic={}, reviveQueueId={}", reviveTopic, queueId); + POP_LOGGER.info("slave skip scan, revive topic={}, reviveQueueId={}", reviveTopic, queueId); continue; } @@ -661,7 +662,7 @@ public void run() { currentReviveMessageTimestamp = System.currentTimeMillis(); } - POP_LOGGER.info("reviveQueueId={},revive finish,old offset is {}, new offset is {}, ckDelay={} ", + POP_LOGGER.info("reviveQueueId={}, revive finish,old offset is {}, new offset is {}, ckDelay={} ", queueId, consumeReviveObj.oldOffset, consumeReviveObj.newOffset, delay); if (sortList == null || sortList.isEmpty()) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java index b2db356c8a4..d3bb048f75d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java @@ -234,7 +234,7 @@ private void handlePushReplyResult(PushReplyResult pushReplyResult, final Remoti } else { response.setCode(ResponseCode.SUCCESS); response.setRemark(null); - //set to zore to avoid client decoding exception + //set to zero to avoid client decoding exception responseHeader.setMsgId("0"); responseHeader.setQueueId(queueIdInt); responseHeader.setQueueOffset(0L); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java index aed0ee19fa5..0c2e6507bd9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java @@ -26,7 +26,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -91,7 +90,7 @@ public class ScheduleMessageService extends ConfigManager { public ScheduleMessageService(final BrokerController brokerController) { this.brokerController = brokerController; this.enableAsyncDeliver = brokerController.getMessageStoreConfig().isEnableScheduleAsyncDeliver(); - scheduledPersistService = new ScheduledThreadPoolExecutor(1, + scheduledPersistService = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("ScheduleMessageServicePersistThread", true, brokerController.getBrokerConfig())); } @@ -134,9 +133,9 @@ public long computeDeliverTimestamp(final int delayLevel, final long storeTimest public void start() { if (started.compareAndSet(false, true)) { this.load(); - this.deliverExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_")); + this.deliverExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_")); if (this.enableAsyncDeliver) { - this.handleExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_")); + this.handleExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_")); } for (Map.Entry entry : this.delayLevelTable.entrySet()) { Integer level = entry.getKey(); @@ -566,7 +565,8 @@ public void run() { pendingQueue.remove(); break; case RUNNING: - break; + scheduleNextTask(); + return; case EXCEPTION: if (!isStarted()) { log.warn("HandlePutResultTask shutdown, info={}", putResultProcess.toString()); @@ -586,6 +586,10 @@ public void run() { } } + scheduleNextTask(); + } + + private void scheduleNextTask() { if (isStarted()) { ScheduleMessageService.this.handleExecutorService .schedule(new HandlePutResultTask(this.delayLevel), DELAY_FOR_A_SLEEP, TimeUnit.MILLISECONDS); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java index 6503970af2e..e9a81a8d686 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java @@ -30,7 +30,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { public RocksDBSubscriptionGroupManager(BrokerController brokerController) { super(brokerController, false); - this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); + this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override @@ -53,7 +53,7 @@ protected SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupCo SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); try { - byte[] keyBytes = groupName.getBytes(DataConverter.charset); + byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8); byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible); this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); } catch (Exception e) { @@ -68,7 +68,7 @@ protected SubscriptionGroupConfig putSubscriptionGroupConfigIfAbsent(Subscriptio SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig); if (oldConfig == null) { try { - byte[] keyBytes = groupName.getBytes(DataConverter.charset); + byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8); byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible); this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); } catch (Exception e) { @@ -82,7 +82,7 @@ protected SubscriptionGroupConfig putSubscriptionGroupConfigIfAbsent(Subscriptio protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) { SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName); try { - this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.charset)); + this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.CHARSET_UTF8)); } catch (Exception e) { log.error("kv delete sub Failed, {}", subscriptionGroupConfig.toString()); } @@ -91,7 +91,7 @@ protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName @Override protected void decode0(byte[] key, byte[] body) { - String groupName = new String(key, DataConverter.charset); + String groupName = new String(key, DataConverter.CHARSET_UTF8); SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class); this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java index 74e39c0fedb..e63b9305868 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -341,10 +341,7 @@ public void deleteSubscriptionGroupConfig(final String groupName) { public void setSubscriptionGroupTable(ConcurrentMap subscriptionGroupTable) { - this.subscriptionGroupTable.clear(); - for (String key : subscriptionGroupTable.keySet()) { - putSubscriptionGroupConfig(subscriptionGroupTable.get(key)); - } + this.subscriptionGroupTable = subscriptionGroupTable; } public boolean containsSubscriptionGroup(String group) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java index 7da0d7c8ac6..fddecf2d92a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java @@ -30,7 +30,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { public RocksDBTopicConfigManager(BrokerController brokerController) { super(brokerController, false); - this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); + this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override @@ -49,7 +49,7 @@ public boolean stop() { @Override protected void decode0(byte[] key, byte[] body) { - String topicName = new String(key, DataConverter.charset); + String topicName = new String(key, DataConverter.CHARSET_UTF8); TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class); this.topicConfigTable.put(topicName, topicConfig); @@ -66,7 +66,7 @@ protected TopicConfig putTopicConfig(TopicConfig topicConfig) { String topicName = topicConfig.getTopicName(); TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig); try { - byte[] keyBytes = topicName.getBytes(DataConverter.charset); + byte[] keyBytes = topicName.getBytes(DataConverter.CHARSET_UTF8); byte[] valueBytes = JSON.toJSONBytes(topicConfig, SerializerFeature.BrowserCompatible); this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); } catch (Exception e) { @@ -79,7 +79,7 @@ protected TopicConfig putTopicConfig(TopicConfig topicConfig) { protected TopicConfig removeTopicConfig(String topicName) { TopicConfig topicConfig = this.topicConfigTable.remove(topicName); try { - this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.charset)); + this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.CHARSET_UTF8)); } catch (Exception e) { log.error("kv remove topic Failed, {}", topicConfig.toString()); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 1c3b9711fda..511d29e12ad 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerPathConfigHelper; @@ -47,7 +48,9 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.protocol.DataVersion; import org.apache.rocketmq.remoting.protocol.body.KVTable; +import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; import static com.google.common.base.Preconditions.checkNotNull; @@ -290,7 +293,7 @@ public TopicConfig createTopicInSendMessageMethod(final String topic, final Stri } if (createNew) { - this.brokerController.registerBrokerAll(false, true, true); + registerBrokerData(topicConfig); } return topicConfig; @@ -330,7 +333,7 @@ public TopicConfig createTopicIfAbsent(TopicConfig topicConfig, boolean register log.error("createTopicIfAbsent ", e); } if (createNew && register) { - this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); + registerBrokerData(topicConfig); } return getTopicConfig(topicConfig.getTopicName()); } @@ -390,7 +393,7 @@ public TopicConfig createTopicInSendMessageBackMethod( } if (createNew) { - this.brokerController.registerBrokerAll(false, true, true); + registerBrokerData(topicConfig); } return topicConfig; @@ -431,7 +434,7 @@ public TopicConfig createTopicOfTranCheckMaxTime(final int clientDefaultTopicQue } if (createNew) { - this.brokerController.registerBrokerAll(false, true, true); + registerBrokerData(topicConfig); } return topicConfig; @@ -457,7 +460,7 @@ public void updateTopicUnitFlag(final String topic, final boolean unit) { dataVersion.nextVersion(stateMachineVersion); this.persist(); - this.brokerController.registerBrokerAll(false, true, true); + registerBrokerData(topicConfig); } } @@ -480,7 +483,7 @@ public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub) dataVersion.nextVersion(stateMachineVersion); this.persist(); - this.brokerController.registerBrokerAll(false, true, true); + registerBrokerData(topicConfig); } } @@ -585,6 +588,24 @@ public TopicConfigSerializeWrapper buildTopicConfigSerializeWrapper() { return topicConfigSerializeWrapper; } + public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper(final ConcurrentMap topicConfigTable) { + return buildSerializeWrapper(topicConfigTable, Maps.newHashMap()); + } + + public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper( + final ConcurrentMap topicConfigTable, + final Map topicQueueMappingInfoMap + ) { + TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); + topicConfigWrapper.setTopicConfigTable(topicConfigTable); + topicConfigWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap); + topicConfigWrapper.setDataVersion(this.getDataVersion()); + if (this.brokerController.getBrokerConfig().isEnableSplitRegistration()) { + this.getDataVersion().nextVersion(); + } + return topicConfigWrapper; + } + @Override public String encode() { return encode(false); @@ -654,6 +675,14 @@ private Map current(String topic) { } } + private void registerBrokerData(TopicConfig topicConfig) { + if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { + this.brokerController.registerSingleTopicAll(topicConfig); + } else { + this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); + } + } + public boolean containsTopic(String topic) { return topicConfigTable.containsKey(topic); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java index b3556472555..11bde5f5fe2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java @@ -23,7 +23,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; @@ -36,6 +35,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -66,7 +66,7 @@ public TopicRouteInfoManager(BrokerController brokerController) { } public void start() { - this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread")); + this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread")); this.scheduledExecutorService.scheduleAtFixedRate(() -> { try { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java index 771d8430060..982355d783f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java @@ -19,7 +19,6 @@ import io.netty.channel.Channel; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; @@ -27,6 +26,7 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; @@ -97,7 +97,7 @@ public void shutDown() { public synchronized void initExecutorService() { if (executorService == null) { - executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), + executorService = ThreadUtils.newThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), new ThreadFactoryImpl("Transaction-msg-check-thread", brokerController.getBrokerIdentity()), new CallerRunsPolicy()); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index 93fa725a93c..48db828e0ae 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -629,7 +629,9 @@ public boolean open() { @Override public void close() { - + if (this.transactionalOpBatchService != null) { + this.transactionalOpBatchService.shutdown(); + } } public Message getOpMessage(int queueId, String moreData) { diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml index 3c51e59d4bc..32dc297360e 100644 --- a/broker/src/main/resources/rmq.broker.logback.xml +++ b/broker/src/main/resources/rmq.broker.logback.xml @@ -672,6 +672,11 @@ + + + + + diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java index 75ad961ce9f..6035a20acb2 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java @@ -23,9 +23,9 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import org.apache.rocketmq.broker.latency.FutureTaskExt; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.future.FutureTaskExt; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.RequestTask; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java index 5d0f7f9d72b..31b547cf1be 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java @@ -19,6 +19,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.future.FutureTaskExt; import org.apache.rocketmq.remoting.netty.RequestTask; import org.junit.Test; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java index d33a217f76d..ec252cecea6 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.LongAdder; import org.apache.rocketmq.broker.BrokerController; @@ -41,6 +42,7 @@ import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.KeyBuilder; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; @@ -74,7 +76,6 @@ import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; -import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.logfile.DefaultMappedFile; import org.apache.rocketmq.store.stats.BrokerStats; @@ -90,8 +91,11 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -245,32 +249,6 @@ public void testUpdateAndCreateTopic() throws Exception { assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } - @Test - public void testUpdateAndCreateTopicOnSlaveInRocksdb() throws Exception { - if (notToBeExecuted()) { - return; - } - initRocksdbTopicManager(); - testUpdateAndCreateTopicOnSlave(); - } - - @Test - public void testUpdateAndCreateTopicOnSlave() throws Exception { - // setup - MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); - when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); - defaultMessageStore = mock(DefaultMessageStore.class); - when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); - - // test on slave - String topic = "TEST_CREATE_TOPIC"; - RemotingCommand request = buildCreateTopicRequest(topic); - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); - assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + - "please execute it from master broker."); - } - @Test public void testDeleteTopicInRocksdb() throws Exception { if (notToBeExecuted()) { @@ -297,28 +275,34 @@ public void testDeleteTopic() throws Exception { } @Test - public void testDeleteTopicOnSlaveInRocksdb() throws Exception { - if (notToBeExecuted()) { - return; - } - initRocksdbTopicManager(); - testDeleteTopicOnSlave(); - } + public void testDeleteWithPopRetryTopic() throws Exception { + String topic = "topicA"; + String anotherTopic = "another_topicA"; - @Test - public void testDeleteTopicOnSlave() throws Exception { - // setup - MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); - when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); - defaultMessageStore = mock(DefaultMessageStore.class); - when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + topicConfigManager = mock(TopicConfigManager.class); + when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); + final ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); + topicConfigTable.put(topic, new TopicConfig()); + topicConfigTable.put(KeyBuilder.buildPopRetryTopic(topic, "cid1"), new TopicConfig()); + + topicConfigTable.put(anotherTopic, new TopicConfig()); + topicConfigTable.put(KeyBuilder.buildPopRetryTopic(anotherTopic, "cid2"), new TopicConfig()); + when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable); + when(topicConfigManager.selectTopicConfig(anyString())).thenAnswer(invocation -> { + final String selectTopic = invocation.getArgument(0); + return topicConfigManager.getTopicConfigTable().get(selectTopic); + }); + + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + when(consumerOffsetManager.whichGroupByTopic(topic)).thenReturn(Sets.newHashSet("cid1")); - String topic = "TEST_DELETE_TOPIC"; RemotingCommand request = buildDeleteTopicRequest(topic); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); - assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + - "please execute it from master broker."); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + verify(topicConfigManager).deleteTopicConfig(topic); + verify(topicConfigManager).deleteTopicConfig(KeyBuilder.buildPopRetryTopic(topic, "cid1")); + verify(messageStore, times(2)).deleteTopics(anySet()); } @Test @@ -502,36 +486,6 @@ public void testUpdateAndCreateSubscriptionGroup() throws RemotingCommandExcepti assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } - @Test - public void testUpdateAndCreateSubscriptionGroupOnSlaveInRocksdb() throws Exception { - initRocksdbSubscriptionManager(); - testUpdateAndCreateSubscriptionGroupOnSlave(); - } - - @Test - public void testUpdateAndCreateSubscriptionGroupOnSlave() throws RemotingCommandException { - // Setup - MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); - when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); - defaultMessageStore = mock(DefaultMessageStore.class); - when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); - - // Test - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null); - SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); - subscriptionGroupConfig.setBrokerId(1); - subscriptionGroupConfig.setGroupName("groupId"); - subscriptionGroupConfig.setConsumeEnable(Boolean.TRUE); - subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.TRUE); - subscriptionGroupConfig.setRetryMaxTimes(111); - subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.TRUE); - request.setBody(JSON.toJSON(subscriptionGroupConfig).toString().getBytes()); - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); - assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + - "please execute it from master broker."); - } - @Test public void testGetAllSubscriptionGroupInRocksdb() throws Exception { initRocksdbSubscriptionManager(); @@ -560,30 +514,6 @@ public void testDeleteSubscriptionGroup() throws RemotingCommandException { assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } - @Test - public void testDeleteSubscriptionGroupOnSlaveInRocksdb() throws Exception { - initRocksdbSubscriptionManager(); - testDeleteSubscriptionGroupOnSlave(); - } - - @Test - public void testDeleteSubscriptionGroupOnSlave() throws RemotingCommandException { - // Setup - MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); - when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); - defaultMessageStore = mock(DefaultMessageStore.class); - when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); - - // Test - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, null); - request.addExtField("groupName", "GID-Group-Name"); - request.addExtField("removeOffset", "true"); - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); - assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + - "please execute it from master broker."); - } - @Test public void testGetTopicStatsInfo() throws RemotingCommandException { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, null); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java index 1c3a0cd459a..78b76264fef 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java @@ -234,7 +234,7 @@ public static MessageExtBrokerInner buildCkMsg(PopCheckPoint ck) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(REVIVE_TOPIC); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(REVIVE_QUEUE_ID); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); @@ -269,7 +269,7 @@ public static MessageExtBrokerInner buildAckInnerMessage(String reviveTopic, Ack SocketAddress host, long deliverMs, String ackUniqueId) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/client/pom.xml b/client/pom.xml index c59a4388994..d6fb3889b7a 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index f87450f66c3..f9843cc0231 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -38,6 +38,8 @@ public class ClientConfig { public static final String SOCKS_PROXY_CONFIG = "com.rocketmq.socks.proxy.config"; public static final String DECODE_READ_BODY = "com.rocketmq.read.body"; public static final String DECODE_DECOMPRESS_BODY = "com.rocketmq.decompress.body"; + public static final String SEND_LATENCY_ENABLE = "com.rocketmq.sendLatencyEnable"; + public static final String START_DETECTOR_ENABLE = "com.rocketmq.startDetectorEnable"; public static final String HEART_BEAT_V2 = "com.rocketmq.heartbeat.v2"; private String namesrvAddr = NameServerAddressUtils.getNameServerAddresses(); private String clientIP = NetworkUtil.getLocalAddress(); @@ -72,6 +74,8 @@ public class ClientConfig { private String socksProxyConfig = System.getProperty(SOCKS_PROXY_CONFIG, "{}"); private int mqClientApiTimeout = 3 * 1000; + private int detectTimeout = 200; + private int detectInterval = 2 * 1000; private LanguageCode language = LanguageCode.JAVA; @@ -81,6 +85,17 @@ public class ClientConfig { */ protected boolean enableStreamRequestType = false; + /** + * Enable the fault tolerance mechanism of the client sending process. + * DO NOT OPEN when ORDER messages are required. + * Turning on will interfere with the queue selection functionality, + * possibly conflicting with the order message. + */ + private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false")); + private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false")); + + private boolean enableHeartbeatChannelEventListener = true; + public String buildMQClientId() { StringBuilder sb = new StringBuilder(); sb.append(this.getClientIP()); @@ -186,6 +201,11 @@ public void resetClientConfig(final ClientConfig cc) { this.decodeDecompressBody = cc.decodeDecompressBody; this.enableStreamRequestType = cc.enableStreamRequestType; this.useHeartbeatV2 = cc.useHeartbeatV2; + this.startDetectorEnable = cc.startDetectorEnable; + this.sendLatencyEnable = cc.sendLatencyEnable; + this.enableHeartbeatChannelEventListener = cc.enableHeartbeatChannelEventListener; + this.detectInterval = cc.detectInterval; + this.detectTimeout = cc.detectTimeout; } public ClientConfig cloneClientConfig() { @@ -210,6 +230,11 @@ public ClientConfig cloneClientConfig() { cc.decodeDecompressBody = decodeDecompressBody; cc.enableStreamRequestType = enableStreamRequestType; cc.useHeartbeatV2 = useHeartbeatV2; + cc.startDetectorEnable = startDetectorEnable; + cc.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; + cc.sendLatencyEnable = sendLatencyEnable; + cc.detectInterval = detectInterval; + cc.detectTimeout = detectTimeout; return cc; } @@ -381,6 +406,46 @@ public void setEnableStreamRequestType(boolean enableStreamRequestType) { this.enableStreamRequestType = enableStreamRequestType; } + public boolean isSendLatencyEnable() { + return sendLatencyEnable; + } + + public void setSendLatencyEnable(boolean sendLatencyEnable) { + this.sendLatencyEnable = sendLatencyEnable; + } + + public boolean isStartDetectorEnable() { + return startDetectorEnable; + } + + public void setStartDetectorEnable(boolean startDetectorEnable) { + this.startDetectorEnable = startDetectorEnable; + } + + public boolean isEnableHeartbeatChannelEventListener() { + return enableHeartbeatChannelEventListener; + } + + public void setEnableHeartbeatChannelEventListener(boolean enableHeartbeatChannelEventListener) { + this.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; + } + + public int getDetectTimeout() { + return this.detectTimeout; + } + + public void setDetectTimeout(int detectTimeout) { + this.detectTimeout = detectTimeout; + } + + public int getDetectInterval() { + return this.detectInterval; + } + + public void setDetectInterval(int detectInterval) { + this.detectInterval = detectInterval; + } + public boolean isUseHeartbeatV2() { return useHeartbeatV2; } @@ -391,18 +456,34 @@ public void setUseHeartbeatV2(boolean useHeartbeatV2) { @Override public String toString() { - return "ClientConfig [namesrvAddr=" + namesrvAddr - + ", clientIP=" + clientIP + ", instanceName=" + instanceName - + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads - + ", pollNameServerInterval=" + pollNameServerInterval - + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval - + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval - + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException - + ", unitMode=" + unitMode + ", unitName=" + unitName - + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS - + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name() - + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout - + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody - + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]"; + return "ClientConfig{" + + "namesrvAddr='" + namesrvAddr + '\'' + + ", clientIP='" + clientIP + '\'' + + ", instanceName='" + instanceName + '\'' + + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + + ", namespace='" + namespace + '\'' + + ", namespaceInitialized=" + namespaceInitialized + + ", accessChannel=" + accessChannel + + ", pollNameServerInterval=" + pollNameServerInterval + + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + + ", unitMode=" + unitMode + + ", unitName='" + unitName + '\'' + + ", decodeReadBody=" + decodeReadBody + + ", decodeDecompressBody=" + decodeDecompressBody + + ", vipChannelEnabled=" + vipChannelEnabled + + ", useHeartbeatV2=" + useHeartbeatV2 + + ", useTLS=" + useTLS + + ", socksProxyConfig='" + socksProxyConfig + '\'' + + ", mqClientApiTimeout=" + mqClientApiTimeout + + ", detectTimeout=" + detectTimeout + + ", detectInterval=" + detectInterval + + ", language=" + language + + ", enableStreamRequestType=" + enableStreamRequestType + + ", sendLatencyEnable=" + sendLatencyEnable + + ", startDetectorEnable=" + startDetectorEnable + + ", enableHeartbeatChannelEventListener=" + enableHeartbeatChannelEventListener + + '}'; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java index 4a3d90135b6..3a086c13dff 100644 --- a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java +++ b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java @@ -33,6 +33,14 @@ public int incrementAndGet() { return index & POSITIVE_MASK; } + public void reset() { + int index = Math.abs(random.nextInt(Integer.MAX_VALUE)); + if (index < 0) { + index = 0; + } + this.threadLocalIndex.set(index); + } + @Override public String toString() { return "ThreadLocalIndex{" + diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index 1ef3a948357..83835bd3d3e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -44,6 +44,8 @@ import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.utils.NetworkUtil; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -55,8 +57,6 @@ import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.route.BrokerData; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public class MQAdminImpl { @@ -357,44 +357,51 @@ protected QueryResult queryMessage(String clusterName, String topic, String key, new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { try { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - switch (response.getCode()) { - case ResponseCode.SUCCESS: { - QueryMessageResponseHeader responseHeader = null; - try { - responseHeader = - (QueryMessageResponseHeader) response - .decodeCommandCustomHeader(QueryMessageResponseHeader.class); - } catch (RemotingCommandException e) { - log.error("decodeCommandCustomHeader exception", e); - return; - } - - List wrappers = - MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); - - QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); - try { - lock.writeLock().lock(); - queryResultList.add(qr); - } finally { - lock.writeLock().unlock(); - } - break; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryMessageResponseHeader responseHeader = null; + try { + responseHeader = + (QueryMessageResponseHeader) response + .decodeCommandCustomHeader(QueryMessageResponseHeader.class); + } catch (RemotingCommandException e) { + log.error("decodeCommandCustomHeader exception", e); + return; } - default: - log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); - break; + + List wrappers = + MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); + + QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); + try { + lock.writeLock().lock(); + queryResultList.add(qr); + } finally { + lock.writeLock().unlock(); + } + break; } - } else { - log.warn("getResponseCommand return null"); + default: + log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); + break; } + } finally { countDownLatch.countDown(); } } + + @Override + public void operationFail(Throwable throwable) { + log.error("queryMessage error, requestHeader={}", requestHeader); + countDownLatch.countDown(); + } }, isUniqKey); } catch (Exception e) { log.warn("queryMessage exception", e); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 5101ffc8e42..e152be81193 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -32,7 +33,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.Validators; -import org.apache.rocketmq.client.common.ClientErrorCode; import org.apache.rocketmq.client.consumer.AckCallback; import org.apache.rocketmq.client.consumer.AckResult; import org.apache.rocketmq.client.consumer.AckStatus; @@ -54,6 +54,7 @@ import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; import org.apache.rocketmq.common.AclConfig; +import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; @@ -76,7 +77,9 @@ import org.apache.rocketmq.common.namesrv.TopAddressing; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; -import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; @@ -101,7 +104,10 @@ import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.BatchAck; +import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; +import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData; import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody; import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo; @@ -114,7 +120,6 @@ import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; -import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; import org.apache.rocketmq.remoting.protocol.body.KVTable; import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; import org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody; @@ -196,6 +201,10 @@ import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader; @@ -207,10 +216,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.PutKVConfigRequestHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData; import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; @@ -221,8 +226,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.remoting.rpchook.DynamicalExtFieldRPCHook; import org.apache.rocketmq.remoting.rpchook.StreamTypeRPCHook; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS; @@ -244,10 +247,16 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, final ClientRemotingProcessor clientRemotingProcessor, RPCHook rpcHook, final ClientConfig clientConfig) { + this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null); + } + + public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, + final ClientRemotingProcessor clientRemotingProcessor, + RPCHook rpcHook, final ClientConfig clientConfig, final ChannelEventListener channelEventListener) { this.clientConfig = clientConfig; topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName()); topAddressing.registerChangeCallBack(this); - this.remotingClient = new NettyRemotingClient(nettyClientConfig, null); + this.remotingClient = new NettyRemotingClient(nettyClientConfig, channelEventListener); this.clientRemotingProcessor = clientRemotingProcessor; // Inject stream rpc hook first to make reserve field signature @@ -650,10 +659,13 @@ private void sendMessageAsync( this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { - long cost = System.currentTimeMillis() - beginStartTime; - RemotingCommand response = responseFuture.getResponseCommand(); - if (null == sendCallback && response != null) { + } + + @Override + public void operationSucceed(RemotingCommand response) { + long cost = System.currentTimeMillis() - beginStartTime; + if (null == sendCallback) { try { SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); if (context != null && sendResult != null) { @@ -663,52 +675,53 @@ public void operationComplete(ResponseFuture responseFuture) { } catch (Throwable e) { } - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); + producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); return; } - if (response != null) { + try { + SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); + assert sendResult != null; + if (context != null) { + context.setSendResult(sendResult); + context.getProducer().executeSendMessageHookAfter(context); + } + try { - SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); - assert sendResult != null; - if (context != null) { - context.setSendResult(sendResult); - context.getProducer().executeSendMessageHookAfter(context); - } + sendCallback.onSuccess(sendResult); + } catch (Throwable e) { + } - try { - sendCallback.onSuccess(sendResult); - } catch (Throwable e) { - } + producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); + } catch (Exception e) { + producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, e, context, false, producer); + } + } - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); - } catch (Exception e) { - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, e, context, false, producer); - } + @Override + public void operationFail(Throwable throwable) { + producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); + long cost = System.currentTimeMillis() - beginStartTime; + if (throwable instanceof RemotingSendRequestException) { + MQClientException ex = new MQClientException("send request failed", throwable); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); + } else if (throwable instanceof RemotingTimeoutException) { + MQClientException ex = new MQClientException("wait response timeout, cost=" + cost, throwable); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); } else { - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); - if (!responseFuture.isSendRequestOK()) { - MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, ex, context, true, producer); - } else if (responseFuture.isTimeout()) { - MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", - responseFuture.getCause()); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, ex, context, true, producer); - } else { - MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause()); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, ex, context, true, producer); - } + MQClientException ex = new MQClientException("unknow reseaon", throwable); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); } } }); } catch (Exception ex) { long cost = System.currentTimeMillis() - beginStartTime; - producer.updateFaultItem(brokerName, cost, true); + producer.updateFaultItem(brokerName, cost, true, false); onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, retryTimesWhenSendFailed, times, ex, context, true, producer); } @@ -732,7 +745,7 @@ private void onExceptionImpl(final String brokerName, if (needRetry && tmp <= timesTotal) { String retryBrokerName = brokerName;//by default, it will send to the same broker if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send - MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName); + MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName, false); retryBrokerName = instance.getBrokerNameFromMessageQueue(mqChosen); } String addr = instance.findBrokerAddressInPublish(retryBrokerName); @@ -854,30 +867,25 @@ public void popMessageAsync( final long timeoutMillis, final PopCallback popCallback ) throws RemotingException, InterruptedException { final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader); - this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { @Override - public void onComplete(ResponseFuture responseFuture) { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - PopResult - popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); - assert popResult != null; - popCallback.onSuccess(popResult); - } catch (Exception e) { - popCallback.onException(e); - } - } else { - if (!responseFuture.isSendRequestOK()) { - popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, - responseFuture.getCause())); - } else { - popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); - } + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + try { + PopResult popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); + popCallback.onSuccess(popResult); + } catch (Exception e) { + popCallback.onException(e); } } + @Override + public void operationFail(Throwable throwable) { + popCallback.onException(throwable); + } }); } @@ -885,37 +893,97 @@ public void ackMessageAsync( final String addr, final long timeOut, final AckCallback ackCallback, - final AckMessageRequestHeader requestHeader // + final AckMessageRequestHeader requestHeader + ) throws RemotingException, MQBrokerException, InterruptedException { + ackMessageAsync(addr, timeOut, ackCallback, requestHeader, null); + } + + public void batchAckMessageAsync( + final String addr, + final long timeOut, + final AckCallback ackCallback, + final String topic, + final String consumerGroup, + final List extraInfoList + ) throws RemotingException, MQBrokerException, InterruptedException { + String brokerName = null; + Map batchAckMap = new HashMap<>(); + for (String extraInfo : extraInfoList) { + String[] extraInfoData = ExtraInfoUtil.split(extraInfo); + if (brokerName == null) { + brokerName = ExtraInfoUtil.getBrokerName(extraInfoData); + } + String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + + ExtraInfoUtil.getQueueId(extraInfoData) + "@" + + ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + + ExtraInfoUtil.getPopTime(extraInfoData); + BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { + BatchAck newBatchAck = new BatchAck(); + newBatchAck.setConsumerGroup(consumerGroup); + newBatchAck.setTopic(topic); + newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); + newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); + newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); + newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); + newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); + newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); + newBatchAck.setBitSet(new BitSet()); + return newBatchAck; + }); + bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); + } + + BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); + requestBody.setBrokerName(brokerName); + requestBody.setAcks(new ArrayList<>(batchAckMap.values())); + batchAckMessageAsync(addr, timeOut, ackCallback, requestBody); + } + + public void batchAckMessageAsync( + final String addr, + final long timeOut, + final AckCallback ackCallback, + final BatchAckMessageRequestBody requestBody ) throws RemotingException, MQBrokerException, InterruptedException { - final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); - this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) { + ackMessageAsync(addr, timeOut, ackCallback, null, requestBody); + } + protected void ackMessageAsync( + final String addr, + final long timeOut, + final AckCallback ackCallback, + final AckMessageRequestHeader requestHeader, + final BatchAckMessageRequestBody requestBody + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request; + if (requestHeader != null) { + request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); + } else { + request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); + if (requestBody != null) { + request.setBody(requestBody.encode()); + } + } + this.remotingClient.invokeAsync(addr, request, timeOut, new InvokeCallback() { @Override - public void onComplete(ResponseFuture responseFuture) { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - AckResult ackResult = new AckResult(); - if (ResponseCode.SUCCESS == response.getCode()) { - ackResult.setStatus(AckStatus.OK); - } else { - ackResult.setStatus(AckStatus.NO_EXIST); - } - ackCallback.onSuccess(ackResult); - } catch (Exception e) { - ackCallback.onException(e); - } + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + AckResult ackResult = new AckResult(); + if (ResponseCode.SUCCESS == response.getCode()) { + ackResult.setStatus(AckStatus.OK); } else { - if (!responseFuture.isSendRequestOK()) { - ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, - responseFuture.getCause())); - } else { - ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause())); - } + ackResult.setStatus(AckStatus.NO_EXIST); } + ackCallback.onSuccess(ackResult); + } + @Override + public void operationFail(Throwable throwable) { + ackCallback.onException(throwable); } }); } @@ -928,39 +996,37 @@ public void changeInvisibleTimeAsync(// final AckCallback ackCallback ) throws RemotingException, MQBrokerException, InterruptedException { final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader); - this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { @Override - public void onComplete(ResponseFuture responseFuture) { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); - AckResult ackResult = new AckResult(); - if (ResponseCode.SUCCESS == response.getCode()) { - ackResult.setStatus(AckStatus.OK); - ackResult.setPopTime(responseHeader.getPopTime()); - ackResult.setExtraInfo(ExtraInfoUtil - .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), - responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR - + requestHeader.getOffset()); - } else { - ackResult.setStatus(AckStatus.NO_EXIST); - } - ackCallback.onSuccess(ackResult); - } catch (Exception e) { - ackCallback.onException(e); - } - } else { - if (!responseFuture.isSendRequestOK()) { - ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, - responseFuture.getCause())); + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + try { + ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); + AckResult ackResult = new AckResult(); + if (ResponseCode.SUCCESS == response.getCode()) { + ackResult.setStatus(AckStatus.OK); + ackResult.setPopTime(responseHeader.getPopTime()); + ackResult.setExtraInfo(ExtraInfoUtil + .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), + responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR + + requestHeader.getOffset()); } else { - ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); + ackResult.setStatus(AckStatus.NO_EXIST); } + ackCallback.onSuccess(ackResult); + } catch (Exception e) { + ackCallback.onException(e); } } + + @Override + public void operationFail(Throwable throwable) { + ackCallback.onException(throwable); + } }); } @@ -973,26 +1039,23 @@ private void pullMessageAsync( this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); - assert pullResult != null; - pullCallback.onSuccess(pullResult); - } catch (Exception e) { - pullCallback.onException(e); - } - } else { - if (!responseFuture.isSendRequestOK()) { - pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, - responseFuture.getCause())); - } else { - pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); - } + + } + + @Override + public void operationSucceed(RemotingCommand response) { + try { + PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); + pullCallback.onSuccess(pullResult); + } catch (Exception e) { + pullCallback.onException(e); } } + + @Override + public void operationFail(Throwable throwable) { + pullCallback.onException(throwable); + } }); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 8851bc81599..09534a1768b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.client.impl.factory; +import io.netty.channel.Channel; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -29,6 +30,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; @@ -64,6 +66,7 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueueAssignment; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.HeartbeatV2Result; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -125,6 +128,12 @@ public class MQClientInstance { private final Set brokerSupportV2HeartbeatSet = new HashSet(); private final ConcurrentMap brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap(); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "MQClientFactoryScheduledThread")); + private final ScheduledExecutorService fetchRemoteConfigExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "MQClientFactoryFetchRemoteConfigScheduledThread"); + } + }); private final PullMessageService pullMessageService; private final RebalanceService rebalanceService; private final DefaultMQProducer defaultMQProducer; @@ -144,7 +153,38 @@ public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String cli this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS()); this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig()); ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this); - this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig); + ChannelEventListener channelEventListener; + if (clientConfig.isEnableHeartbeatChannelEventListener()) { + channelEventListener = new ChannelEventListener() { + private final ConcurrentMap> brokerAddrTable = MQClientInstance.this.brokerAddrTable; + @Override + public void onChannelConnect(String remoteAddr, Channel channel) { + for (Map.Entry> addressEntry : brokerAddrTable.entrySet()) { + for (String address : addressEntry.getValue().values()) { + if (address.equals(remoteAddr)) { + sendHeartbeatToAllBrokerWithLockV2(false); + break; + } + } + } + } + + @Override + public void onChannelClose(String remoteAddr, Channel channel) { + } + + @Override + public void onChannelException(String remoteAddr, Channel channel) { + } + + @Override + public void onChannelIdle(String remoteAddr, Channel channel) { + } + }; + } else { + channelEventListener = null; + } + this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, channelEventListener); if (this.clientConfig.getNamesrvAddr() != null) { this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr()); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java index fb8f8d11fd4..f3102e17597 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java @@ -30,7 +30,6 @@ import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.exception.OffsetNotFoundException; import org.apache.rocketmq.client.impl.ClientRemotingProcessor; import org.apache.rocketmq.client.impl.CommunicationMode; @@ -47,6 +46,7 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; @@ -106,19 +106,6 @@ public boolean updateNameServerAddressList() { return false; } - protected static MQClientException processNullResponseErr(ResponseFuture responseFuture) { - MQClientException ex; - if (!responseFuture.isSendRequestOK()) { - ex = new MQClientException("send request failed", responseFuture.getCause()); - } else if (responseFuture.isTimeout()) { - ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", - responseFuture.getCause()); - } else { - ex = new MQClientException("unknown reason", responseFuture.getCause()); - } - return ex; - } - public CompletableFuture sendHeartbeatOneway( String brokerAddr, HeartbeatData heartbeatData, @@ -146,24 +133,15 @@ public CompletableFuture sendHeartbeatAsync( request.setLanguage(clientConfig.getLanguage()); request.setBody(heartbeatData.encode()); - CompletableFuture future = new CompletableFuture<>(); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (ResponseCode.SUCCESS == response.getCode()) { - future.complete(response.getVersion()); - } else { - future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); - } - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); - } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture future0 = new CompletableFuture<>(); + if (ResponseCode.SUCCESS == response.getCode()) { + future0.complete(response.getVersion()); + } else { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); + } + return future0; + }); } public CompletableFuture sendMessageAsync( @@ -177,24 +155,15 @@ public CompletableFuture sendMessageAsync( RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); request.setBody(msg.getBody()); - CompletableFuture future = new CompletableFuture<>(); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - future.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); - } catch (Exception e) { - future.completeExceptionally(e); - } - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); - } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture future0 = new CompletableFuture<>(); + try { + future0.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); + } catch (Exception e) { + future0.completeExceptionally(e); + } + return future0; + }); } public CompletableFuture sendMessageAsync( @@ -216,17 +185,14 @@ public CompletableFuture sendMessageAsync( msgBatch.setBody(body); request.setBody(body); - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - future.complete(this.processSendResponse(brokerName, msgBatch, response, brokerAddr)); - } catch (Exception e) { - future.completeExceptionally(e); - } - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture future0 = new CompletableFuture<>(); + try { + future0.complete(processSendResponse(brokerName, msgBatch, response, brokerAddr)); + } catch (Exception e) { + future0.completeExceptionally(e); } + return future0; }); } catch (Throwable t) { future.completeExceptionally(t); @@ -240,21 +206,7 @@ public CompletableFuture sendMessageBackAsync( long timeoutMillis ) { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); - - CompletableFuture future = new CompletableFuture<>(); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - future.complete(response); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); - } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis); } public CompletableFuture popMessageAsync( @@ -306,6 +258,32 @@ public void onException(Throwable t) { return future; } + public CompletableFuture batchAckMessageAsync( + String brokerAddr, + String topic, + String consumerGroup, + List extraInfoList, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() { + @Override + public void onSuccess(AckResult ackResult) { + future.complete(ackResult); + } + + @Override + public void onException(Throwable t) { + future.completeExceptionally(t); + } + }, topic, consumerGroup, extraInfoList); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + public CompletableFuture changeInvisibleTimeAsync( String brokerAddr, String brokerName, @@ -376,38 +354,31 @@ public CompletableFuture queryConsumerOffsetWithFuture( QueryConsumerOffsetRequestHeader requestHeader, long timeoutMillis ) { - CompletableFuture future = new CompletableFuture<>(); - try { - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - switch (response.getCode()) { - case ResponseCode.SUCCESS: { - try { - QueryConsumerOffsetResponseHeader responseHeader = - (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); - future.complete(responseHeader.getOffset()); - } catch (RemotingCommandException e) { - future.completeExceptionally(e); - } - break; - } - case ResponseCode.QUERY_NOT_FOUND: { - future.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); - break; - } - default: - break; + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture future0 = new CompletableFuture<>(); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + try { + QueryConsumerOffsetResponseHeader responseHeader = + (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); + future0.complete(responseHeader.getOffset()); + } catch (RemotingCommandException e) { + future0.completeExceptionally(e); } - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + break; } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + case ResponseCode.QUERY_NOT_FOUND: { + future0.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); + break; + } + default: { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + break; + } + } + return future0; + }); } public CompletableFuture updateConsumerOffsetOneWay( @@ -435,9 +406,14 @@ public CompletableFuture> getConsumerListByGroupAsync( CompletableFuture> future = new CompletableFuture<>(); try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { switch (response.getCode()) { case ResponseCode.SUCCESS: { if (response.getBody() != null) { @@ -459,8 +435,11 @@ public CompletableFuture> getConsumerListByGroupAsync( break; } future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } }); } catch (Throwable t) { @@ -475,9 +454,14 @@ public CompletableFuture getMaxOffset(String brokerAddr, GetMaxOffsetReque CompletableFuture future = new CompletableFuture<>(); try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { if (ResponseCode.SUCCESS == response.getCode()) { try { GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); @@ -487,8 +471,11 @@ public CompletableFuture getMaxOffset(String brokerAddr, GetMaxOffsetReque } } future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } }); } catch (Throwable t) { @@ -503,9 +490,14 @@ public CompletableFuture getMinOffset(String brokerAddr, GetMinOffsetReque CompletableFuture future = new CompletableFuture<>(); try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { if (ResponseCode.SUCCESS == response.getCode()) { try { GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); @@ -515,8 +507,11 @@ public CompletableFuture getMinOffset(String brokerAddr, GetMinOffsetReque } } future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } }); } catch (Throwable t) { @@ -529,57 +524,41 @@ public CompletableFuture searchOffset(String brokerAddr, SearchOffsetReque long timeoutMillis) { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); - CompletableFuture future = new CompletableFuture<>(); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (response.getCode() == ResponseCode.SUCCESS) { - try { - SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); - future.complete(responseHeader.getOffset()); - } catch (Throwable t) { - future.completeExceptionally(t); - } - } - future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture future0 = new CompletableFuture<>(); + if (response.getCode() == ResponseCode.SUCCESS) { + try { + SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); + future0.complete(responseHeader.getOffset()); + } catch (Throwable t) { + future0.completeExceptionally(t); } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + } else { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } + return future0; + }); } public CompletableFuture> lockBatchMQWithFuture(String brokerAddr, LockBatchRequestBody requestBody, long timeoutMillis) { - CompletableFuture> future = new CompletableFuture<>(); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); request.setBody(requestBody.encode()); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (response.getCode() == ResponseCode.SUCCESS) { - try { - LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); - Set messageQueues = responseBody.getLockOKMQSet(); - future.complete(messageQueues); - } catch (Throwable t) { - future.completeExceptionally(t); - } - } - future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture> future0 = new CompletableFuture<>(); + if (response.getCode() == ResponseCode.SUCCESS) { + try { + LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); + Set messageQueues = responseBody.getLockOKMQSet(); + future0.complete(messageQueues); + } catch (Throwable t) { + future0.completeExceptionally(t); } - }); - } catch (Exception e) { - future.completeExceptionally(e); - } - return future; + } else { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } + return future0; + }); } public CompletableFuture unlockBatchMQOneway(String brokerAddr, @@ -598,25 +577,21 @@ public CompletableFuture unlockBatchMQOneway(String brokerAddr, public CompletableFuture notification(String brokerAddr, NotificationRequestHeader requestHeader, long timeoutMillis) { - CompletableFuture future = new CompletableFuture<>(); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader); - try { - this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenAccept(response -> { - if (response.getCode() == ResponseCode.SUCCESS) { - try { - NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); - future.complete(responseHeader.isHasMsg()); - } catch (Throwable t) { - future.completeExceptionally(t); - } - } else { - future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture future0 = new CompletableFuture<>(); + if (response.getCode() == ResponseCode.SUCCESS) { + try { + NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); + future0.complete(responseHeader.isHasMsg()); + } catch (Throwable t) { + future0.completeExceptionally(t); } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + } else { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } + return future0; + }); } public CompletableFuture invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 3f4c6e5f7a4..b0c212e46b6 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -33,6 +33,8 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; + +import com.google.common.base.Optional; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; @@ -49,6 +51,8 @@ import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.latency.MQFaultStrategy; +import org.apache.rocketmq.client.latency.Resolver; +import org.apache.rocketmq.client.latency.ServiceDetector; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; @@ -112,7 +116,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { private ServiceState serviceState = ServiceState.CREATE_JUST; private MQClientInstance mQClientFactory; private ArrayList checkForbiddenHookList = new ArrayList<>(); - private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); + private MQFaultStrategy mqFaultStrategy; private ExecutorService asyncSenderExecutor; // compression related @@ -153,8 +157,38 @@ public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook semaphoreAsyncSendSize = new Semaphore(1024 * 1024, true); log.info("semaphoreAsyncSendSize can not be smaller than 1M."); } - } + ServiceDetector serviceDetector = new ServiceDetector() { + @Override + public boolean detect(String endpoint, long timeoutMillis) { + Optional candidateTopic = pickTopic(); + if (!candidateTopic.isPresent()) { + return false; + } + try { + MessageQueue mq = new MessageQueue(candidateTopic.get(), null, 0); + mQClientFactory.getMQClientAPIImpl() + .getMaxOffset(endpoint, mq, timeoutMillis); + return true; + } catch (Exception e) { + return false; + } + } + }; + + this.mqFaultStrategy = new MQFaultStrategy(defaultMQProducer.cloneClientConfig(), new Resolver() { + @Override + public String resolve(String name) { + return DefaultMQProducerImpl.this.mQClientFactory.findBrokerAddressInPublish(name); + } + }, serviceDetector); + } + private Optional pickTopic() { + if (topicPublishInfoTable.isEmpty()) { + return Optional.absent(); + } + return Optional.of(topicPublishInfoTable.keySet().iterator().next()); + } public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { this.checkForbiddenHookList.add(checkForbiddenHook); log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), @@ -229,6 +263,8 @@ public void start(final boolean startFactory) throws MQClientException { mQClientFactory.start(); } + this.mqFaultStrategy.startDetector(); + log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), this.defaultMQProducer.isSendMessageWithVIPChannel()); this.serviceState = ServiceState.RUNNING; @@ -273,6 +309,7 @@ public void shutdown(final boolean shutdownFactory) { if (shutdownFactory) { this.mQClientFactory.shutdown(); } + this.mqFaultStrategy.shutdown(); RequestFutureHolder.getInstance().shutdown(this); log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); this.serviceState = ServiceState.SHUTDOWN_ALREADY; @@ -506,6 +543,8 @@ public void send(Message msg, @Deprecated public void send(final Message msg, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { + BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); + final long beginStartTime = System.currentTimeMillis(); Runnable runnable = new Runnable() { @Override @@ -513,20 +552,53 @@ public void run() { long costTime = System.currentTimeMillis() - beginStartTime; if (timeout > costTime) { try { - sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); + sendDefaultImpl(msg, CommunicationMode.ASYNC, newCallBack, timeout - costTime); } catch (Exception e) { - sendCallback.onException(e); + newCallBack.onException(e); } } else { - sendCallback.onException( + newCallBack.onException( new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); } } }; - executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); + executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); } - public void executeAsyncMessageSend(Runnable runnable, final Message msg, final SendCallback sendCallback, + class BackpressureSendCallBack implements SendCallback { + public boolean isSemaphoreAsyncSizeAquired = false; + public boolean isSemaphoreAsyncNumAquired = false; + public int msgLen; + private final SendCallback sendCallback; + + public BackpressureSendCallBack(final SendCallback sendCallback) { + this.sendCallback = sendCallback; + } + + @Override + public void onSuccess(SendResult sendResult) { + if (isSemaphoreAsyncSizeAquired) { + semaphoreAsyncSendSize.release(msgLen); + } + if (isSemaphoreAsyncNumAquired) { + semaphoreAsyncSendNum.release(); + } + sendCallback.onSuccess(sendResult); + } + + @Override + public void onException(Throwable e) { + if (isSemaphoreAsyncSizeAquired) { + semaphoreAsyncSendSize.release(msgLen); + } + if (isSemaphoreAsyncNumAquired) { + semaphoreAsyncSendNum.release(); + } + sendCallback.onException(e); + } + } + + public void executeAsyncMessageSend(Runnable runnable, final Message msg, final BackpressureSendCallBack sendCallback, final long timeout, final long beginStartTime) throws MQClientException, InterruptedException { ExecutorService executor = this.getAsyncSenderExecutor(); @@ -554,7 +626,9 @@ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final return; } } - + sendCallback.isSemaphoreAsyncSizeAquired = isSemaphoreAsyncSizeAquired; + sendCallback.isSemaphoreAsyncNumAquired = isSemaphoreAsyncNumAquired; + sendCallback.msgLen = msgLen; executor.submit(runnable); } catch (RejectedExecutionException e) { if (isEnableBackpressureForAsyncMode) { @@ -562,19 +636,11 @@ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final } else { throw new MQClientException("executor rejected ", e); } - } finally { - if (isSemaphoreAsyncSizeAquired) { - semaphoreAsyncSendSize.release(msgLen); - } - if (isSemaphoreAsyncNumAquired) { - semaphoreAsyncSendNum.release(); - } } - } public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg, - final long timeout) throws MQClientException, RemotingTooMuchRequestException { + final long timeout) throws MQClientException, RemotingTooMuchRequestException { long beginStartTime = System.currentTimeMillis(); this.makeSureStateOK(); Validators.checkMessage(msg, this.defaultMQProducer); @@ -584,7 +650,7 @@ public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector MessageQueue mq = null; try { List messageQueueList = - mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); + mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); Message userMessage = MessageAccessor.cloneMessage(msg); String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace()); userMessage.setTopic(userTopic); @@ -609,12 +675,13 @@ public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); } - public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { - return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); + public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { + return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName, resetIndex); } - public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { - this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); + public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, + boolean reachable) { + this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); } private void validateNameServerSetting() throws MQClientException { @@ -647,9 +714,13 @@ private SendResult sendDefaultImpl( int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; int times = 0; String[] brokersSent = new String[timesTotal]; + boolean resetIndex = false; for (; times < timesTotal; times++) { String lastBrokerName = null == mq ? null : mq.getBrokerName(); - MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); + if (times > 0) { + resetIndex = true; + } + MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName, resetIndex); if (mqSelected != null) { mq = mqSelected; brokersSent[times] = mq.getBrokerName(); @@ -667,7 +738,7 @@ private SendResult sendDefaultImpl( sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); switch (communicationMode) { case ASYNC: return null; @@ -684,9 +755,22 @@ private SendResult sendDefaultImpl( default: break; } - } catch (RemotingException | MQClientException e) { + } catch (MQClientException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); + log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); + log.warn(msg.toString()); + exception = e; + continue; + } catch (RemotingException e) { endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + if (this.mqFaultStrategy.isStartDetectorEnable()) { + // Set this broker unreachable when detecting schedule task is running for RemotingException. + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false); + } else { + // Otherwise, isolate this broker. + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, true); + } log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); if (log.isDebugEnabled()) { log.debug(msg.toString()); @@ -695,7 +779,7 @@ private SendResult sendDefaultImpl( continue; } catch (MQBrokerException e) { endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false); log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); if (log.isDebugEnabled()) { log.debug(msg.toString()); @@ -712,7 +796,7 @@ private SendResult sendDefaultImpl( } } catch (InterruptedException e) { endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); log.warn("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); if (log.isDebugEnabled()) { log.debug(msg.toString()); @@ -1129,7 +1213,7 @@ public void send(Message msg, MessageQueue mq, SendCallback sendCallback) @Deprecated public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { - + BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); final long beginStartTime = System.currentTimeMillis(); Runnable runnable = new Runnable() { @Override @@ -1144,22 +1228,22 @@ public void run() { long costTime = System.currentTimeMillis() - beginStartTime; if (timeout > costTime) { try { - sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, + sendKernelImpl(msg, mq, CommunicationMode.ASYNC, newCallBack, null, timeout - costTime); } catch (MQBrokerException e) { throw new MQClientException("unknown exception", e); } } else { - sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); + newCallBack.onException(new RemotingTooMuchRequestException("call timeout")); } } catch (Exception e) { - sendCallback.onException(e); + newCallBack.onException(e); } } }; - executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); + executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); } /** @@ -1256,7 +1340,7 @@ public void send(Message msg, MessageQueueSelector selector, Object arg, SendCal public void send(final Message msg, final MessageQueueSelector selector, final Object arg, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { - + BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); final long beginStartTime = System.currentTimeMillis(); Runnable runnable = new Runnable() { @Override @@ -1265,21 +1349,21 @@ public void run() { if (timeout > costTime) { try { try { - sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, + sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, newCallBack, timeout - costTime); } catch (MQBrokerException e) { throw new MQClientException("unknown exception", e); } } catch (Exception e) { - sendCallback.onException(e); + newCallBack.onException(e); } } else { - sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); + newCallBack.onException(new RemotingTooMuchRequestException("call timeout")); } } }; - executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); + executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); } /** diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java index 275ada7ac34..37b1f3252f7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java @@ -18,6 +18,8 @@ import java.util.ArrayList; import java.util.List; + +import com.google.common.base.Preconditions; import org.apache.rocketmq.client.common.ThreadLocalIndex; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.protocol.route.QueueData; @@ -30,6 +32,10 @@ public class TopicPublishInfo { private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex(); private TopicRouteData topicRouteData; + public interface QueueFilter { + boolean filter(MessageQueue mq); + } + public boolean isOrderTopic() { return orderTopic; } @@ -66,6 +72,40 @@ public void setHaveTopicRouterInfo(boolean haveTopicRouterInfo) { this.haveTopicRouterInfo = haveTopicRouterInfo; } + public MessageQueue selectOneMessageQueue(QueueFilter ...filter) { + return selectOneMessageQueue(this.messageQueueList, this.sendWhichQueue, filter); + } + + private MessageQueue selectOneMessageQueue(List messageQueueList, ThreadLocalIndex sendQueue, QueueFilter ...filter) { + if (messageQueueList == null || messageQueueList.isEmpty()) { + return null; + } + + if (filter != null && filter.length != 0) { + for (int i = 0; i < messageQueueList.size(); i++) { + int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); + MessageQueue mq = messageQueueList.get(index); + boolean filterResult = true; + for (QueueFilter f: filter) { + Preconditions.checkNotNull(f); + filterResult &= f.filter(mq); + } + if (filterResult) { + return mq; + } + } + + return null; + } + + int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); + return messageQueueList.get(index); + } + + public void resetIndex() { + this.sendWhichQueue.reset(); + } + public MessageQueue selectOneMessageQueue(final String lastBrokerName) { if (lastBrokerName == null) { return selectOneMessageQueue(); diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java index 09a8aa46189..17aaa266aae 100644 --- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java @@ -18,11 +18,89 @@ package org.apache.rocketmq.client.latency; public interface LatencyFaultTolerance { - void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration); + /** + * Update brokers' states, to decide if they are good or not. + * + * @param name Broker's name. + * @param currentLatency Current message sending process's latency. + * @param notAvailableDuration Corresponding not available time, ms. The broker will be not available until it + * spends such time. + * @param reachable To decide if this broker is reachable or not. + */ + void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration, + final boolean reachable); + /** + * To check if this broker is available. + * + * @param name Broker's name. + * @return boolean variable, if this is true, then the broker is available. + */ boolean isAvailable(final T name); + /** + * To check if this broker is reachable. + * + * @param name Broker's name. + * @return boolean variable, if this is true, then the broker is reachable. + */ + boolean isReachable(final T name); + + /** + * Remove the broker in this fault item table. + * + * @param name broker's name. + */ void remove(final T name); + /** + * The worst situation, no broker can be available. Then choose random one. + * + * @return A random mq will be returned. + */ T pickOneAtLeast(); + + /** + * Start a new thread, to detect the broker's reachable tag. + */ + void startDetector(); + + /** + * Shutdown threads that started by LatencyFaultTolerance. + */ + void shutdown(); + + /** + * A function reserved, just detect by once, won't create a new thread. + */ + void detectByOneRound(); + + /** + * Use it to set the detect timeout bound. + * + * @param detectTimeout timeout bound + */ + void setDetectTimeout(final int detectTimeout); + + /** + * Use it to set the detector's detector interval for each broker (each broker will be detected once during this + * time) + * + * @param detectInterval each broker's detecting interval + */ + void setDetectInterval(final int detectInterval); + + /** + * Use it to set the detector work or not. + * + * @param startDetectorEnable set the detector's work status + */ + void setStartDetectorEnable(final boolean startDetectorEnable); + + /** + * Use it to judge if the detector enabled. + * + * @return is the detector should be started. + */ + boolean isStartDetectorEnable(); } diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java index 93795d95753..d3ff7eb45a1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java @@ -21,30 +21,101 @@ import java.util.Enumeration; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.client.common.ThreadLocalIndex; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public class LatencyFaultToleranceImpl implements LatencyFaultTolerance { - private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap<>(16); + private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class); + private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap(16); + private int detectTimeout = 200; + private int detectInterval = 2000; + private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex(); - private final ThreadLocalIndex randomItem = new ThreadLocalIndex(); + private volatile boolean startDetectorEnable = false; + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "LatencyFaultToleranceScheduledThread"); + } + }); + + private final Resolver resolver; + + private final ServiceDetector serviceDetector; + + public LatencyFaultToleranceImpl(Resolver resolver, ServiceDetector serviceDetector) { + this.resolver = resolver; + this.serviceDetector = serviceDetector; + } + + public void detectByOneRound() { + for (Map.Entry item : this.faultItemTable.entrySet()) { + FaultItem brokerItem = item.getValue(); + if (System.currentTimeMillis() - brokerItem.checkStamp >= 0) { + brokerItem.checkStamp = System.currentTimeMillis() + this.detectInterval; + String brokerAddr = resolver.resolve(brokerItem.getName()); + if (brokerAddr == null) { + faultItemTable.remove(item.getKey()); + continue; + } + if (null == serviceDetector) { + continue; + } + boolean serviceOK = serviceDetector.detect(brokerAddr, detectTimeout); + if (serviceOK && !brokerItem.reachableFlag) { + log.info(brokerItem.name + " is reachable now, then it can be used."); + brokerItem.reachableFlag = true; + } + } + } + } + + public void startDetector() { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + if (startDetectorEnable) { + detectByOneRound(); + } + } catch (Exception e) { + log.warn("Unexpected exception raised while detecting service reachability", e); + } + } + }, 3, 3, TimeUnit.SECONDS); + } + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } @Override - public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration) { + public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration, + final boolean reachable) { FaultItem old = this.faultItemTable.get(name); if (null == old) { final FaultItem faultItem = new FaultItem(name); faultItem.setCurrentLatency(currentLatency); - faultItem.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); - + faultItem.updateNotAvailableDuration(notAvailableDuration); + faultItem.setReachable(reachable); old = this.faultItemTable.putIfAbsent(name, faultItem); - if (old != null) { - old.setCurrentLatency(currentLatency); - old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); - } - } else { + } + + if (null != old) { old.setCurrentLatency(currentLatency); - old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); + old.updateNotAvailableDuration(notAvailableDuration); + old.setReachable(reachable); + } + + if (!reachable) { + log.info(name + " is unreachable, it will not be used until it's reachable"); } } @@ -57,76 +128,121 @@ public boolean isAvailable(final String name) { return true; } + public boolean isReachable(final String name) { + final FaultItem faultItem = this.faultItemTable.get(name); + if (faultItem != null) { + return faultItem.isReachable(); + } + return true; + } + @Override public void remove(final String name) { this.faultItemTable.remove(name); } + public boolean isStartDetectorEnable() { + return startDetectorEnable; + } + + public void setStartDetectorEnable(boolean startDetectorEnable) { + this.startDetectorEnable = startDetectorEnable; + } @Override public String pickOneAtLeast() { final Enumeration elements = this.faultItemTable.elements(); - List tmpList = new LinkedList<>(); + List tmpList = new LinkedList(); while (elements.hasMoreElements()) { final FaultItem faultItem = elements.nextElement(); tmpList.add(faultItem); } + if (!tmpList.isEmpty()) { - Collections.sort(tmpList); - final int half = tmpList.size() / 2; - if (half <= 0) { - return tmpList.get(0).getName(); - } else { - final int i = this.randomItem.incrementAndGet() % half; - return tmpList.get(i).getName(); + Collections.shuffle(tmpList); + for (FaultItem faultItem : tmpList) { + if (faultItem.reachableFlag) { + return faultItem.name; + } } } + return null; } @Override public String toString() { return "LatencyFaultToleranceImpl{" + - "faultItemTable=" + faultItemTable + - ", whichItemWorst=" + randomItem + - '}'; + "faultItemTable=" + faultItemTable + + ", whichItemWorst=" + whichItemWorst + + '}'; } - class FaultItem implements Comparable { + public void setDetectTimeout(final int detectTimeout) { + this.detectTimeout = detectTimeout; + } + + public void setDetectInterval(final int detectInterval) { + this.detectInterval = detectInterval; + } + + public class FaultItem implements Comparable { private final String name; private volatile long currentLatency; private volatile long startTimestamp; + private volatile long checkStamp; + private volatile boolean reachableFlag; public FaultItem(final String name) { this.name = name; } + public void updateNotAvailableDuration(long notAvailableDuration) { + if (notAvailableDuration > 0 && System.currentTimeMillis() + notAvailableDuration > this.startTimestamp) { + this.startTimestamp = System.currentTimeMillis() + notAvailableDuration; + log.info(name + " will be isolated for " + notAvailableDuration + " ms."); + } + } + @Override public int compareTo(final FaultItem other) { if (this.isAvailable() != other.isAvailable()) { - if (this.isAvailable()) + if (this.isAvailable()) { return -1; + } - if (other.isAvailable()) + if (other.isAvailable()) { return 1; + } } - if (this.currentLatency < other.currentLatency) + if (this.currentLatency < other.currentLatency) { return -1; - else if (this.currentLatency > other.currentLatency) { + } else if (this.currentLatency > other.currentLatency) { return 1; } - if (this.startTimestamp < other.startTimestamp) + if (this.startTimestamp < other.startTimestamp) { return -1; - else if (this.startTimestamp > other.startTimestamp) { + } else if (this.startTimestamp > other.startTimestamp) { return 1; } - return 0; } + public void setReachable(boolean reachableFlag) { + this.reachableFlag = reachableFlag; + } + + public void setCheckStamp(long checkStamp) { + this.checkStamp = checkStamp; + } + public boolean isAvailable() { - return (System.currentTimeMillis() - startTimestamp) >= 0; + return reachableFlag && System.currentTimeMillis() >= startTimestamp; + } + + public boolean isReachable() { + return reachableFlag; } @Override @@ -139,28 +255,32 @@ public int hashCode() { @Override public boolean equals(final Object o) { - if (this == o) + if (this == o) { return true; - if (!(o instanceof FaultItem)) + } + if (!(o instanceof FaultItem)) { return false; + } final FaultItem faultItem = (FaultItem) o; - if (getCurrentLatency() != faultItem.getCurrentLatency()) + if (getCurrentLatency() != faultItem.getCurrentLatency()) { return false; - if (getStartTimestamp() != faultItem.getStartTimestamp()) + } + if (getStartTimestamp() != faultItem.getStartTimestamp()) { return false; + } return getName() != null ? getName().equals(faultItem.getName()) : faultItem.getName() == null; - } @Override public String toString() { return "FaultItem{" + - "name='" + name + '\'' + - ", currentLatency=" + currentLatency + - ", startTimestamp=" + startTimestamp + - '}'; + "name='" + name + '\'' + + ", currentLatency=" + currentLatency + + ", startTimestamp=" + startTimestamp + + ", reachableFlag=" + reachableFlag + + '}'; } public String getName() { diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java index 1e1953fad90..69fb533e5ad 100644 --- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java @@ -17,25 +17,86 @@ package org.apache.rocketmq.client.latency; -import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo.QueueFilter; import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public class MQFaultStrategy { - private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class); - private final LatencyFaultTolerance latencyFaultTolerance = new LatencyFaultToleranceImpl(); + private LatencyFaultTolerance latencyFaultTolerance; + private volatile boolean sendLatencyFaultEnable; + private volatile boolean startDetectorEnable; + private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L}; + private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L}; - private boolean sendLatencyFaultEnable = false; + public static class BrokerFilter implements QueueFilter { + private String lastBrokerName; + + public void setLastBrokerName(String lastBrokerName) { + this.lastBrokerName = lastBrokerName; + } + + @Override public boolean filter(MessageQueue mq) { + if (lastBrokerName != null) { + return !mq.getBrokerName().equals(lastBrokerName); + } + return true; + } + } + + private ThreadLocal threadBrokerFilter = new ThreadLocal() { + @Override protected BrokerFilter initialValue() { + return new BrokerFilter(); + } + }; + + private QueueFilter reachableFilter = new QueueFilter() { + @Override public boolean filter(MessageQueue mq) { + return latencyFaultTolerance.isReachable(mq.getBrokerName()); + } + }; + + private QueueFilter availableFilter = new QueueFilter() { + @Override public boolean filter(MessageQueue mq) { + return latencyFaultTolerance.isAvailable(mq.getBrokerName()); + } + }; + + + public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) { + this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector); + this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); + this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); + this.setStartDetectorEnable(cc.isStartDetectorEnable()); + this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); + } + + // For unit test. + public MQFaultStrategy(ClientConfig cc, LatencyFaultTolerance tolerance) { + this.setStartDetectorEnable(cc.isStartDetectorEnable()); + this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); + this.latencyFaultTolerance = tolerance; + this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); + this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); + } - private long[] latencyMax = {50L, 100L, 550L, 1000L, 2000L, 3000L, 15000L}; - private long[] notAvailableDuration = {0L, 0L, 30000L, 60000L, 120000L, 180000L, 600000L}; public long[] getNotAvailableDuration() { return notAvailableDuration; } + public QueueFilter getAvailableFilter() { + return availableFilter; + } + + public QueueFilter getReachableFilter() { + return reachableFilter; + } + + public ThreadLocal getThreadBrokerFilter() { + return threadBrokerFilter; + } + public void setNotAvailableDuration(final long[] notAvailableDuration) { this.notAvailableDuration = notAvailableDuration; } @@ -56,51 +117,63 @@ public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { this.sendLatencyFaultEnable = sendLatencyFaultEnable; } - public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { + public boolean isStartDetectorEnable() { + return startDetectorEnable; + } + + public void setStartDetectorEnable(boolean startDetectorEnable) { + this.startDetectorEnable = startDetectorEnable; + this.latencyFaultTolerance.setStartDetectorEnable(startDetectorEnable); + } + + public void startDetector() { + this.latencyFaultTolerance.startDetector(); + } + + public void shutdown() { + this.latencyFaultTolerance.shutdown(); + } + + public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { + BrokerFilter brokerFilter = threadBrokerFilter.get(); + brokerFilter.setLastBrokerName(lastBrokerName); if (this.sendLatencyFaultEnable) { - try { - int index = tpInfo.getSendWhichQueue().incrementAndGet(); - for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) { - int pos = index++ % tpInfo.getMessageQueueList().size(); - MessageQueue mq = tpInfo.getMessageQueueList().get(pos); - if (!StringUtils.equals(lastBrokerName, mq.getBrokerName()) && latencyFaultTolerance.isAvailable(mq.getBrokerName())) { - return mq; - } - } - - final String notBestBroker = latencyFaultTolerance.pickOneAtLeast(); - int writeQueueNums = tpInfo.getWriteQueueNumsByBroker(notBestBroker); - if (writeQueueNums > 0) { - final MessageQueue mq = tpInfo.selectOneMessageQueue(); - if (notBestBroker != null) { - mq.setBrokerName(notBestBroker); - mq.setQueueId(tpInfo.getSendWhichQueue().incrementAndGet() % writeQueueNums); - } - return mq; - } else { - latencyFaultTolerance.remove(notBestBroker); - } - } catch (Exception e) { - log.error("Error occurred when selecting message queue", e); + if (resetIndex) { + tpInfo.resetIndex(); + } + MessageQueue mq = tpInfo.selectOneMessageQueue(availableFilter, brokerFilter); + if (mq != null) { + return mq; + } + + mq = tpInfo.selectOneMessageQueue(reachableFilter, brokerFilter); + if (mq != null) { + return mq; } return tpInfo.selectOneMessageQueue(); } - return tpInfo.selectOneMessageQueue(lastBrokerName); + MessageQueue mq = tpInfo.selectOneMessageQueue(brokerFilter); + if (mq != null) { + return mq; + } + return tpInfo.selectOneMessageQueue(); } - public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { + public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, + final boolean reachable) { if (this.sendLatencyFaultEnable) { - long duration = computeNotAvailableDuration(isolation ? 30000 : currentLatency); - this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration); + long duration = computeNotAvailableDuration(isolation ? 10000 : currentLatency); + this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration, reachable); } } private long computeNotAvailableDuration(final long currentLatency) { for (int i = latencyMax.length - 1; i >= 0; i--) { - if (currentLatency >= latencyMax[i]) + if (currentLatency >= latencyMax[i]) { return this.notAvailableDuration[i]; + } } return 0; diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java new file mode 100644 index 00000000000..1c29ba33469 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.latency; + +public interface Resolver { + + String resolve(String name); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java new file mode 100644 index 00000000000..c6ffbad1cb6 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.latency; + +/** + * Detect whether the remote service state is normal. + */ +public interface ServiceDetector { + + /** + * Check if the remote service is normal. + * @param endpoint Service endpoint to check against + * @return true if the service is back to normal; false otherwise. + */ + boolean detect(String endpoint, long timeoutMillis); +} diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index d13f2cfe43a..c152d38ea50 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -212,7 +212,7 @@ public Object answer(InvocationOnMock mock) throws Throwable { RemotingCommand request = mock.getArgument(1); ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -386,7 +386,7 @@ public Object answer(InvocationOnMock mock) throws Throwable { RemotingCommand request = mock.getArgument(1); ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class)); @@ -472,7 +472,7 @@ public Void answer(InvocationOnMock mock) throws Throwable { message.putUserProperty("key", "value"); response.setBody(MessageDecoder.encode(message, false)); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -543,7 +543,7 @@ public Void answer(InvocationOnMock mock) throws Throwable { message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0)); response.setBody(MessageDecoder.encode(message, false)); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -585,7 +585,7 @@ public Void answer(InvocationOnMock mock) throws Throwable { response.setOpaque(request.getOpaque()); response.setCode(ResponseCode.SUCCESS); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -622,7 +622,7 @@ public Void answer(InvocationOnMock mock) throws Throwable { responseHeader.setPopTime(System.currentTimeMillis()); responseHeader.setInvisibleTime(10 * 1000L); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); diff --git a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java index 86690e40be6..42ccdae5a48 100644 --- a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java @@ -16,11 +16,14 @@ */ package org.apache.rocketmq.client.latency; -import java.util.concurrent.TimeUnit; +import org.awaitility.core.ThrowingRunnable; import org.junit.Before; import org.junit.Test; +import java.util.concurrent.TimeUnit; + import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; public class LatencyFaultToleranceImplTest { private LatencyFaultTolerance latencyFaultTolerance; @@ -29,28 +32,31 @@ public class LatencyFaultToleranceImplTest { @Before public void init() { - latencyFaultTolerance = new LatencyFaultToleranceImpl(); + latencyFaultTolerance = new LatencyFaultToleranceImpl(null, null); } @Test public void testUpdateFaultItem() throws Exception { - latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000); + latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true); assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); assertThat(latencyFaultTolerance.isAvailable(anotherBrokerName)).isTrue(); } @Test public void testIsAvailable() throws Exception { - latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50); + latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50, true); assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); - TimeUnit.MILLISECONDS.sleep(70); - assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); + await().atMost(500, TimeUnit.MILLISECONDS).untilAsserted(new ThrowingRunnable() { + @Override public void run() throws Throwable { + assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); + } + }); } @Test public void testRemove() throws Exception { - latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000); + latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true); assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); latencyFaultTolerance.remove(brokerName); assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); @@ -58,10 +64,20 @@ public void testRemove() throws Exception { @Test public void testPickOneAtLeast() throws Exception { - latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000); + latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true); assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); - latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000); - assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); + // Bad case, since pickOneAtLeast's behavior becomes random + // latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, "127.0.0.1:12011", true); + // assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); + } + + @Test + public void testIsReachable() throws Exception { + latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true); + assertThat(latencyFaultTolerance.isReachable(brokerName)).isEqualTo(true); + + latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, false); + assertThat(latencyFaultTolerance.isReachable(anotherBrokerName)).isEqualTo(false); } } \ No newline at end of file diff --git a/common/BUILD.bazel b/common/BUILD.bazel index a95a19ccd42..9a0c31e772f 100644 --- a/common/BUILD.bazel +++ b/common/BUILD.bazel @@ -35,11 +35,12 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_sdk", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:org_apache_commons_commons_lang3", "@maven//:org_lz4_lz4_java", "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", - "@maven//:io_github_aliyunmq_rocketmq_rocksdb", + "@maven//:org_apache_rocketmq_rocketmq_rocksdb", ], ) diff --git a/common/pom.xml b/common/pom.xml index 31eb0f087da..a28ed228fd4 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 @@ -80,6 +80,10 @@ io.opentelemetry opentelemetry-sdk + + io.opentelemetry + opentelemetry-exporter-logging-otlp + io.grpc grpc-stub @@ -105,7 +109,7 @@ rocketmq-logback-classic - io.github.aliyunmq + org.apache.rocketmq rocketmq-rocksdb diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 45d26b29cba..0d248c4e170 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -396,6 +396,14 @@ public class BrokerConfig extends BrokerIdentity { private boolean enableMixedMessageType = false; + /** + * This flag and deleteTopicWithBrokerRegistration flag in the NameServer cannot be set to true at the same time, + * otherwise there will be a loss of routing + */ + private boolean enableSplitRegistration = false; + + private int splitRegistrationSize = 800; + public long getMaxPopPollingSize() { return maxPopPollingSize; } @@ -1731,4 +1739,20 @@ public boolean isEnableMixedMessageType() { public void setEnableMixedMessageType(boolean enableMixedMessageType) { this.enableMixedMessageType = enableMixedMessageType; } + + public boolean isEnableSplitRegistration() { + return enableSplitRegistration; + } + + public void setEnableSplitRegistration(boolean enableSplitRegistration) { + this.enableSplitRegistration = enableSplitRegistration; + } + + public int getSplitRegistrationSize() { + return splitRegistrationSize; + } + + public void setSplitRegistrationSize(int splitRegistrationSize) { + this.splitRegistrationSize = splitRegistrationSize; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index bfd07a8959c..4f1990ff828 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ public class MQVersion { - public static final int CURRENT_VERSION = Version.V5_1_3.ordinal(); + public static final int CURRENT_VERSION = Version.V5_1_4.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 1233a54223b..407ef2842ca 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -492,6 +492,7 @@ public static int compareInteger(int x, int y) { public static int compareLong(long x, long y) { return Long.compare(x, y); } + public static boolean isLmq(String lmqMetaData) { return lmqMetaData != null && lmqMetaData.startsWith(LMQ_PREFIX); } diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java index 73ef2188009..9148d5a18aa 100644 --- a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java +++ b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java @@ -19,5 +19,6 @@ public enum CQType { SimpleCQ, - BatchCQ + BatchCQ, + RocksDBCQ } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index e3673baad05..20319abba3d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -17,22 +17,21 @@ package org.apache.rocketmq.common.config; import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.rocksdb.ColumnFamilyDescriptor; @@ -47,7 +46,6 @@ import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; import org.rocksdb.Statistics; import org.rocksdb.Status; import org.rocksdb.WriteBatch; @@ -58,7 +56,6 @@ public abstract class AbstractRocksDBStorage { protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); - private static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); private static final String SPACE = " | "; protected String dbPath; @@ -82,8 +79,8 @@ public abstract class AbstractRocksDBStorage { private volatile boolean closed; private final Semaphore reloadPermit = new Semaphore(1); - private final ScheduledExecutorService reloadScheduler = new ScheduledThreadPoolExecutor(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); - private final ThreadPoolExecutor manualCompactionThread = new ThreadPoolExecutor( + private final ScheduledExecutorService reloadScheduler = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); + private final ThreadPoolExecutor manualCompactionThread = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( 1, 1, 1000 * 60, TimeUnit.MILLISECONDS, new ArrayBlockingQueue(1), new ThreadFactoryImpl("RocksDBManualCompactionService_"), @@ -223,10 +220,6 @@ protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, By } } - protected WrappedRocksIterator newIterator(ColumnFamilyHandle cfHandle, ReadOptions readOptions) { - return new WrappedRocksIterator(this.db.newIterator(cfHandle, readOptions)); - } - protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, final byte[] startKey, final byte[] endKey) throws RocksDBException { if (!hold()) { @@ -243,46 +236,6 @@ protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOption } } - protected void manualCompactionDefaultCfMaxLevel(final CompactionOptions compactionOptions) throws Exception { - final ColumnFamilyHandle defaultCFHandle = this.defaultCFHandle; - final byte[] defaultCFName = defaultCFHandle.getName(); - List fileMetaDataList = this.db.getLiveFilesMetaData(); - if (fileMetaDataList == null || fileMetaDataList.isEmpty()) { - return; - } - - List defaultLiveFileDataList = Lists.newArrayList(); - List inputFileNames = Lists.newArrayList(); - int maxLevel = 0; - for (LiveFileMetaData fileMetaData : fileMetaDataList) { - if (compareTo(fileMetaData.columnFamilyName(), defaultCFName) != 0) { - continue; - } - defaultLiveFileDataList.add(fileMetaData); - if (fileMetaData.level() > maxLevel) { - maxLevel = fileMetaData.level(); - } - } - if (maxLevel == 0) { - LOGGER.info("manualCompactionDefaultCfFiles skip level 0."); - return; - } - - for (LiveFileMetaData fileMetaData : defaultLiveFileDataList) { - if (fileMetaData.level() != maxLevel || fileMetaData.beingCompacted()) { - continue; - } - inputFileNames.add(fileMetaData.path() + fileMetaData.fileName()); - } - if (!inputFileNames.isEmpty()) { - List outputLists = this.db.compactFiles(compactionOptions, defaultCFHandle, - inputFileNames, maxLevel, -1, null); - LOGGER.info("manualCompactionDefaultCfFiles OK. src: {}, dst: {}", inputFileNames, outputLists); - } else { - LOGGER.info("manualCompactionDefaultCfFiles Empty."); - } - } - protected void manualCompactionDefaultCfRange(CompactRangeOptions compactRangeOptions) { if (!hold()) { return; @@ -385,8 +338,10 @@ public synchronized boolean shutdown() { this.options.close(); } //4. close db. - if (db != null) { + if (db != null && !this.readOnly) { this.db.syncWal(); + } + if (db != null) { this.db.closeE(); } //5. help gc. @@ -492,50 +447,6 @@ public void flushWAL() throws RocksDBException { this.db.flushWal(true); } - protected class WrappedRocksIterator { - private final RocksIterator iterator; - - public WrappedRocksIterator(final RocksIterator iterator) { - this.iterator = iterator; - } - - public byte[] key() { - return iterator.key(); - } - - public byte[] value() { - return iterator.value(); - } - - public void next() { - iterator.next(); - } - - public void prev() { - iterator.prev(); - } - - public void seek(byte[] target) { - iterator.seek(target); - } - - public void seekForPrev(byte[] target) { - iterator.seekForPrev(target); - } - - public void seekToFirst() { - iterator.seekToFirst(); - } - - public boolean isValid() { - return iterator.isValid(); - } - - public void close() { - iterator.close(); - } - } - private String getStatusError(RocksDBException e) { if (e == null || e.getStatus() == null) { return "null"; @@ -572,7 +483,7 @@ public void statRocksdb(Logger logger) { sb = new StringBuilder(256); map.put(metaData.level(), sb); } - sb.append(new String(metaData.columnFamilyName(), CHARSET_UTF8)).append(SPACE). + sb.append(new String(metaData.columnFamilyName(), DataConverter.CHARSET_UTF8)).append(SPACE). append(metaData.fileName()).append(SPACE). append("s: ").append(metaData.size()).append(SPACE). append("a: ").append(metaData.numEntries()).append(SPACE). @@ -593,21 +504,4 @@ public void statRocksdb(Logger logger) { } catch (Exception ignored) { } } - - public int compareTo(byte[] v1, byte[] v2) { - int len1 = v1.length; - int len2 = v2.length; - int lim = Math.min(len1, len2); - - int k = 0; - while (k < lim) { - byte c1 = v1[k]; - byte c2 = v2[k]; - if (c1 != c2) { - return c1 - c2; - } - k++; - } - return len1 - len2; - } } \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java index 9d05ed28289..b40f8046e84 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java @@ -60,6 +60,12 @@ public ConfigRocksDBStorage(final String dbPath) { this.readOnly = false; } + public ConfigRocksDBStorage(final String dbPath, boolean readOnly) { + super(); + this.dbPath = dbPath; + this.readOnly = readOnly; + } + private void initOptions() { this.options = createConfigDBOptions(); @@ -197,7 +203,7 @@ private DBOptions createConfigDBOptions() { setUseDirectReads(true); } - private static String getDBLogDir() { + public static String getDBLogDir() { String rootPath = System.getProperty("user.home"); if (StringUtils.isEmpty(rootPath)) { return ""; diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java index cb04b00b3ec..61310893f43 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java @@ -21,6 +21,7 @@ public class LoggerName { public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; + public static final String CONTROLLER_CONSOLE_NAME = "RocketmqControllerConsole"; public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java index e02b526a184..c7997c47318 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java @@ -218,14 +218,36 @@ public String toString() { public void setDelayTimeSec(long sec) { this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC, String.valueOf(sec)); } + + public long getDelayTimeSec() { + String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC); + if (t != null) { + return Long.parseLong(t); + } + return 0; + } + public void setDelayTimeMs(long timeMs) { this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_MS, String.valueOf(timeMs)); } + + public long getDelayTimeMs() { + String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS); + if (t != null) { + return Long.parseLong(t); + } + return 0; + } + public void setDeliverTimeMs(long timeMs) { this.putProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(timeMs)); } public long getDeliverTimeMs() { - return Long.parseLong(this.getUserProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS)); + String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS); + if (t != null) { + return Long.parseLong(t); + } + return 0; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java index 0c72ebb7bbd..91599653c5f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java @@ -70,4 +70,17 @@ public MessageVersion getVersion() { public void setVersion(MessageVersion version) { this.version = version; } + + public void removeWaitStorePropertyString() { + if (this.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) { + // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message. + // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it. + String waitStoreMsgOKValue = this.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); + this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties())); + // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later + this.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue); + } else { + this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties())); + } + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java index 411da922192..7b68873a99f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java +++ b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java @@ -29,7 +29,8 @@ public class FutureTaskExtThreadPoolExecutor extends ThreadPoolExecutor { public FutureTaskExtThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, - BlockingQueue workQueue, ThreadFactory threadFactory, + BlockingQueue workQueue, + ThreadFactory threadFactory, RejectedExecutionHandler handler) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); } diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java index 49d97a5d723..1bfabbffedd 100644 --- a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java +++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java @@ -22,12 +22,12 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -36,7 +36,7 @@ public class ThreadPoolMonitor { private static Logger waterMarkLogger = LoggerFactory.getLogger(ThreadPoolMonitor.class); private static final List MONITOR_EXECUTOR = new CopyOnWriteArrayList<>(); - private static final ScheduledExecutorService MONITOR_SCHEDULED = Executors.newSingleThreadScheduledExecutor( + private static final ScheduledExecutorService MONITOR_SCHEDULED = ThreadUtils.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("ThreadPoolMonitor-%d").build() ); @@ -81,7 +81,7 @@ public static ThreadPoolExecutor createAndMonitor(int corePoolSize, String name, int queueCapacity, List threadPoolStatusMonitors) { - ThreadPoolExecutor executor = new FutureTaskExtThreadPoolExecutor( + ThreadPoolExecutor executor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( corePoolSize, maximumPoolSize, keepAliveTime, diff --git a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java index 61265c05d7c..c19592a44c3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java +++ b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java @@ -31,6 +31,7 @@ public class TopicValidator { public static final String RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC"; public static final String RMQ_SYS_SELF_TEST_TOPIC = "SELF_TEST_TOPIC"; public static final String RMQ_SYS_OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; + public static final String RMQ_SYS_ROCKSDB_OFFSET_TOPIC = "CHECKPOINT_TOPIC"; public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; public static final String SYNC_BROKER_MEMBER_GROUP_PREFIX = SYSTEM_TOPIC_PREFIX + "SYNC_BROKER_MEMBER_"; @@ -55,6 +56,7 @@ public class TopicValidator { SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC); SYSTEM_TOPIC_SET.add(RMQ_SYS_SELF_TEST_TOPIC); SYSTEM_TOPIC_SET.add(RMQ_SYS_OFFSET_MOVED_EVENT); + SYSTEM_TOPIC_SET.add(RMQ_SYS_ROCKSDB_OFFSET_TOPIC); NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC); NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_TRANS_HALF_TOPIC); diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java index 421adaca4da..7b4b24819c6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java @@ -43,4 +43,21 @@ public static String generateMd5(byte[] content) { byte[] bytes = calculateMd5(content); return Hex.encodeHexString(bytes, false); } + + /** + * Returns true if subject contains only bytes that are spec-compliant ASCII characters. + * @param subject + * @return + */ + public static boolean isAscii(byte[] subject) { + if (subject == null) { + return false; + } + for (byte b : subject) { + if ((b & 0x80) != 0) { + return false; + } + } + return true; + } } \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java index 8b50de12be8..cc96770b22a 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java @@ -20,7 +20,7 @@ import java.nio.charset.Charset; public class DataConverter { - public static Charset charset = Charset.forName("UTF-8"); + public static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); public static byte[] Long2Byte(Long v) { ByteBuffer tmp = ByteBuffer.allocate(8); diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java index 4b366d4e39b..1644c6360ec 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java @@ -20,38 +20,94 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.thread.FutureTaskExtThreadPoolExecutor; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public final class ThreadUtils { private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME); - public static ExecutorService newThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, - TimeUnit unit, BlockingQueue workQueue, String processName, boolean isDaemon) { - return new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon)); + public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) { + return ThreadUtils.newSingleThreadExecutor(newThreadFactory(processName, isDaemon)); } - public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) { - return Executors.newSingleThreadExecutor(newThreadFactory(processName, isDaemon)); + public static ExecutorService newSingleThreadExecutor(ThreadFactory threadFactory) { + return ThreadUtils.newThreadPoolExecutor(1, threadFactory); + } + + public static ExecutorService newThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory) { + return ThreadUtils.newThreadPoolExecutor(corePoolSize, corePoolSize, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), + threadFactory); + } + + public static ExecutorService newThreadPoolExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, BlockingQueue workQueue, + String processName, + boolean isDaemon) { + return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon)); + } + + public static ExecutorService newThreadPoolExecutor(final int corePoolSize, + final int maximumPoolSize, + final long keepAliveTime, + final TimeUnit unit, + final BlockingQueue workQueue, + final ThreadFactory threadFactory) { + return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, new ThreadPoolExecutor.AbortPolicy()); + } + + public static ExecutorService newThreadPoolExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory, + RejectedExecutionHandler handler) { + return new FutureTaskExtThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); } public static ScheduledExecutorService newSingleThreadScheduledExecutor(String processName, boolean isDaemon) { - return Executors.newSingleThreadScheduledExecutor(newThreadFactory(processName, isDaemon)); + return ThreadUtils.newScheduledThreadPool(1, processName, isDaemon); + } + + public static ScheduledExecutorService newSingleThreadScheduledExecutor(ThreadFactory threadFactory) { + return ThreadUtils.newScheduledThreadPool(1, threadFactory); + } + + public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize) { + return ThreadUtils.newScheduledThreadPool(corePoolSize, Executors.defaultThreadFactory()); } - public static ScheduledExecutorService newFixedThreadScheduledPool(int nThreads, String processName, + public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, String processName, boolean isDaemon) { - return Executors.newScheduledThreadPool(nThreads, newThreadFactory(processName, isDaemon)); + return ThreadUtils.newScheduledThreadPool(corePoolSize, newThreadFactory(processName, isDaemon)); + } + + public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, ThreadFactory threadFactory) { + return ThreadUtils.newScheduledThreadPool(corePoolSize, threadFactory, new ThreadPoolExecutor.AbortPolicy()); + } + + public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, + ThreadFactory threadFactory, + RejectedExecutionHandler handler) { + return new ScheduledThreadPoolExecutor(corePoolSize, threadFactory, handler); } public static ThreadFactory newThreadFactory(String processName, boolean isDaemon) { - return newGenericThreadFactory("Remoting-" + processName, isDaemon); + return newGenericThreadFactory("ThreadUtils-" + processName, isDaemon); } public static ThreadFactory newGenericThreadFactory(String processName) { diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java index f568a65f4d5..a0653d7fc43 100644 --- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java @@ -238,41 +238,54 @@ public void testCalculateFileSizeInPath() throws Exception { */ String basePath = System.getProperty("java.io.tmpdir") + File.separator + "testCalculateFileSizeInPath"; File baseFile = new File(basePath); - // test empty path - assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile)); - - // create baseDir - assertTrue(baseFile.mkdirs()); - - File file0 = new File(baseFile, "file_0"); - assertTrue(file0.createNewFile()); - writeFixedBytesToFile(file0, 1313); - - assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile)); - - // build a file tree like above - File dir1 = new File(baseFile, "dir_1"); - dir1.mkdirs(); - File file10 = new File(dir1, "file_1_0"); - File file11 = new File(dir1, "file_1_1"); - File dir12 = new File(dir1, "dir_1_2"); - dir12.mkdirs(); - File file120 = new File(dir12, "file_1_2_0"); - File dir2 = new File(baseFile, "dir_2"); - dir2.mkdirs(); - - // write all file with 1313 bytes data - assertTrue(file10.createNewFile()); - writeFixedBytesToFile(file10, 1313); - assertTrue(file11.createNewFile()); - writeFixedBytesToFile(file11, 1313); - assertTrue(file120.createNewFile()); - writeFixedBytesToFile(file120, 1313); - - assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile)); - - // clear all file - baseFile.deleteOnExit(); + try { + // test empty path + assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile)); + + // create baseDir + assertTrue(baseFile.mkdirs()); + + File file0 = new File(baseFile, "file_0"); + assertTrue(file0.createNewFile()); + writeFixedBytesToFile(file0, 1313); + + assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile)); + + // build a file tree like above + File dir1 = new File(baseFile, "dir_1"); + dir1.mkdirs(); + File file10 = new File(dir1, "file_1_0"); + File file11 = new File(dir1, "file_1_1"); + File dir12 = new File(dir1, "dir_1_2"); + dir12.mkdirs(); + File file120 = new File(dir12, "file_1_2_0"); + File dir2 = new File(baseFile, "dir_2"); + dir2.mkdirs(); + + // write all file with 1313 bytes data + assertTrue(file10.createNewFile()); + writeFixedBytesToFile(file10, 1313); + assertTrue(file11.createNewFile()); + writeFixedBytesToFile(file11, 1313); + assertTrue(file120.createNewFile()); + writeFixedBytesToFile(file120, 1313); + + assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile)); + } finally { + deleteFolder(baseFile); + } + } + + public static void deleteFolder(File folder) { + if (folder.isDirectory()) { + File[] files = folder.listFiles(); + if (files != null) { + for (File file : files) { + deleteFolder(file); + } + } + } + folder.delete(); } private void writeFixedBytesToFile(File file, int size) throws Exception { diff --git a/container/pom.xml b/container/pom.xml index c8499f12730..8af231e013e 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -18,7 +18,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java index c6446f058fa..5b712bc30db 100644 --- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java +++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java @@ -47,14 +47,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; public class BrokerContainer implements IBrokerContainer { private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); - private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, + private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, new BasicThreadFactory.Builder() .namingPattern("BrokerContainerScheduledThread") .daemon(true) @@ -143,7 +141,7 @@ public boolean initialize() { this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.containerClientHouseKeepingService); this.fastRemotingServer = this.remotingServer.newRemotingServer(this.nettyServerConfig.getListenPort() - 2); - this.brokerContainerExecutor = new ThreadPoolExecutor( + this.brokerContainerExecutor = ThreadUtils.newThreadPoolExecutor( 1, 1, 1000 * 60, diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java index 2ac69112d76..5b825fe811c 100644 --- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java +++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java @@ -91,11 +91,10 @@ private synchronized RemotingCommand addBroker(ChannelHandlerContext ctx, LOGGER.error("addBroker load config from {} failed, {}", configPath, e); } } else { - byte[] body = request.getBody(); - if (body != null) { - String bodyStr = new String(body, MixAll.DEFAULT_CHARSET); - brokerProperties = MixAll.string2Properties(bodyStr); - } + LOGGER.error("addBroker config path is empty"); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("addBroker config path is empty"); + return response; } if (brokerProperties == null) { diff --git a/controller/BUILD.bazel b/controller/BUILD.bazel index 843d9dc7766..b2b743eb2d3 100644 --- a/controller/BUILD.bazel +++ b/controller/BUILD.bazel @@ -49,6 +49,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", "@maven//:io_opentelemetry_opentelemetry_exporter_logging", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:org_slf4j_jul_to_slf4j", ], ) diff --git a/controller/pom.xml b/controller/pom.xml index 3346c7c8254..8432b220bee 100644 --- a/controller/pom.xml +++ b/controller/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 jar diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java index 7c91e70da50..3e6b0eba517 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java @@ -25,8 +25,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RunnableFuture; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; @@ -34,8 +32,8 @@ import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.future.FutureTaskExt; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy; import org.apache.rocketmq.controller.impl.DLedgerController; import org.apache.rocketmq.controller.impl.heartbeat.DefaultBrokerHeartbeatManager; @@ -93,18 +91,14 @@ public ControllerManager(ControllerConfig controllerConfig, NettyServerConfig ne public boolean initialize() { this.controllerRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.controllerConfig.getControllerRequestThreadPoolQueueCapacity()); - this.controllerRequestExecutor = new ThreadPoolExecutor( + this.controllerRequestExecutor = ThreadUtils.newThreadPoolExecutor( this.controllerConfig.getControllerThreadPoolNums(), this.controllerConfig.getControllerThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.controllerRequestThreadPoolQueue, - new ThreadFactoryImpl("ControllerRequestExecutorThread_")) { - @Override - protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { - return new FutureTaskExt(runnable, value); - } - }; + new ThreadFactoryImpl("ControllerRequestExecutorThread_")); + this.notifyService.initialize(); if (StringUtils.isEmpty(this.controllerConfig.getControllerDLegerPeers())) { throw new IllegalArgumentException("Attribute value controllerDLegerPeers of ControllerConfig is null or empty"); diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java index 401720d0507..9e96a704de1 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java @@ -94,9 +94,10 @@ public static ControllerManager createControllerManager(String[] args) throws IO } if (commandLine.hasOption('p')) { - MixAll.printObjectProperties(null, controllerConfig); - MixAll.printObjectProperties(null, nettyServerConfig); - MixAll.printObjectProperties(null, nettyClientConfig); + Logger console = LoggerFactory.getLogger(LoggerName.CONTROLLER_CONSOLE_NAME); + MixAll.printObjectProperties(console, controllerConfig); + MixAll.printObjectProperties(console, nettyServerConfig); + MixAll.printObjectProperties(console, nettyClientConfig); System.exit(0); } diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java index fa91f288e2d..33e4406e402 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java @@ -32,7 +32,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -44,6 +43,7 @@ import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.controller.Controller; import org.apache.rocketmq.controller.elect.ElectPolicy; import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy; @@ -66,11 +66,11 @@ import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.body.SyncStateSet; import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; import org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader; import org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader; import org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader; @@ -136,7 +136,7 @@ public DLedgerController(final ControllerConfig controllerConfig, this.dLedgerServer = new DLedgerServer(dLedgerConfig, nettyServerConfig, nettyClientConfig, channelEventListener); this.dLedgerServer.registerStateMachine(this.statemachine); this.dLedgerServer.getDLedgerLeaderElector().addRoleChangeHandler(this.roleHandler); - this.scanInactiveMasterService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_")); + this.scanInactiveMasterService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_")); this.brokerLifecycleListeners = new ArrayList<>(); } @@ -513,7 +513,7 @@ public void handleException(final Throwable t) { class RoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler { private final String selfId; - private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_")); + private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_")); private volatile MemberState.Role currentRole = MemberState.Role.FOLLOWER; public RoleChangeHandler(final String selfId) { diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java index 2fbddb9cdf9..6ebb2c99420 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java @@ -31,6 +31,7 @@ import org.apache.rocketmq.common.ControllerConfig; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.controller.BrokerHeartbeatManager; import org.apache.rocketmq.controller.helper.BrokerLifecycleListener; import org.apache.rocketmq.logging.org.slf4j.Logger; @@ -66,7 +67,7 @@ public void shutdown() { @Override public void initialize() { - this.scheduledService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_")); + this.scheduledService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_")); this.executor = Executors.newFixedThreadPool(2, new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_executorService_")); } diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java index b0a67531da2..d83a690f908 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java @@ -104,7 +104,7 @@ public ControllerResult alterSyncStateSet( } // Check master - if (!syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) { + if (syncStateInfo.getMasterBrokerId() == null || !syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) { String err = String.format("Rejecting alter syncStateSet request because the current leader is:{%s}, not {%s}", syncStateInfo.getMasterBrokerId(), request.getMasterBrokerId()); LOGGER.error("{}", err); diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java index 650740bcc6e..be9e77eeaeb 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java @@ -26,7 +26,7 @@ import io.opentelemetry.api.metrics.LongUpDownCounter; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.ObservableLongGauge; -import io.opentelemetry.exporter.logging.LoggingMetricExporter; +import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; @@ -38,6 +38,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; import io.opentelemetry.sdk.metrics.View; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.export.MetricExporter; import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; import io.opentelemetry.sdk.resources.Resource; import java.io.File; @@ -121,7 +122,7 @@ public class ControllerMetricsManager { private PrometheusHttpServer prometheusHttpServer; - private LoggingMetricExporter loggingMetricExporter; + private MetricExporter loggingMetricExporter; public static ControllerMetricsManager getInstance(ControllerManager controllerManager) { if (instance == null) { @@ -364,8 +365,8 @@ public void init() { if (type == MetricsExporterType.LOG) { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); - loggingMetricExporter = LoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); - java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); + java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) .setInterval(config.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) .build(); diff --git a/controller/src/main/resources/rmq.controller.logback.xml b/controller/src/main/resources/rmq.controller.logback.xml index bb158213af6..18083e8f987 100644 --- a/controller/src/main/resources/rmq.controller.logback.xml +++ b/controller/src/main/resources/rmq.controller.logback.xml @@ -116,6 +116,10 @@ + + + + diff --git a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java index 595a5cb6536..d6e5449c51b 100644 --- a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java +++ b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java @@ -63,7 +63,8 @@ public class DLedgerControllerTest { private List baseDirs; private List controllers; - public DLedgerController launchController(final String group, final String peers, final String selfId, final boolean isEnableElectUncleanMaster) { + public DLedgerController launchController(final String group, final String peers, final String selfId, + final boolean isEnableElectUncleanMaster) { String tmpdir = System.getProperty("java.io.tmpdir"); final String path = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + group + File.separator + selfId; baseDirs.add(path); @@ -121,11 +122,11 @@ public void registerNewBroker(Controller leader, String clusterName, String brok final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, nextBrokerId, brokerAddress); RemotingCommand remotingCommand2 = leader.registerBroker(registerBrokerToControllerRequestHeader).get(2, TimeUnit.SECONDS); - assertEquals(ResponseCode.SUCCESS, remotingCommand2.getCode()); } - public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, Long brokerId, + public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, + Long brokerId, boolean exceptSuccess) throws Exception { final ElectMasterRequestHeader electMasterRequestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); RemotingCommand command = leader.electMaster(electMasterRequestHeader).get(2, TimeUnit.SECONDS); @@ -186,9 +187,9 @@ public DLedgerController mockMetaData(boolean enableElectUncleanMaster) throws E registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L); registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L); // try elect - brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L,true); - brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); - brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L,false); + brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, true); + brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); + brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, false); final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS); final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader(); assertEquals(1, replicaInfo.getMasterEpoch().intValue()); @@ -239,6 +240,8 @@ public void testElectMaster() throws Exception { @Test public void testBrokerLifecycleListener() throws Exception { final DLedgerController leader = mockMetaData(false); + + assertTrue(leader.isLeaderState()); // Mock that master broker has been inactive, and try to elect a new master from sync-state-set // But we shut down two controller, so the ElectMasterEvent will be appended to DLedger failed. // So the statemachine still keep the stale master's information @@ -247,15 +250,20 @@ public void testBrokerLifecycleListener() throws Exception { dLedgerController.shutdown(); controllers.remove(dLedgerController); } + final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME); setBrokerElectPolicy(leader, 1L); Exception exception = null; + RemotingCommand remotingCommand = null; try { - leader.electMaster(request).get(5, TimeUnit.SECONDS); + remotingCommand = leader.electMaster(request).get(5, TimeUnit.SECONDS); } catch (Exception e) { exception = e; } - assertNotNull(exception); + + assertTrue(exception != null || + remotingCommand != null && remotingCommand.getCode() == ResponseCode.CONTROLLER_NOT_LEADER); + // Shut down leader controller leader.shutdown(); controllers.remove(leader); @@ -272,7 +280,7 @@ public void testBrokerLifecycleListener() throws Exception { setBrokerAlivePredicate(newLeader, 1L); // Check if the statemachine is stale final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)). - get(10, TimeUnit.SECONDS); + get(10, TimeUnit.SECONDS); final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader(); assertEquals(1, replicaInfo.getMasterBrokerId().longValue()); assertEquals(1, replicaInfo.getMasterEpoch().intValue()); diff --git a/distribution/bin/mqbroker b/distribution/bin/mqbroker index 3758ed597c0..35eb93c4461 100644 --- a/distribution/bin/mqbroker +++ b/distribution/bin/mqbroker @@ -68,11 +68,11 @@ if [ "$enable_proxy" = true ]; then if [ "$broker_config" != "" ]; then args_for_proxy=${args_for_proxy}" -bc "${broker_config} fi - sh ${ROCKETMQ_HOME}/bin/runserver.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy} + sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy} else args_for_broker=$other_args if [ "$broker_config" != "" ]; then args_for_broker=${args_for_broker}" -c "${broker_config} fi sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.broker.logback.xml org.apache.rocketmq.broker.BrokerStartup ${args_for_broker} -fi \ No newline at end of file +fi diff --git a/distribution/pom.xml b/distribution/pom.xml index dbde2d9d4a7..73474d34a9e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/docs/cn/Debug_In_Idea.md b/docs/cn/Debug_In_Idea.md new file mode 100644 index 00000000000..fd01751ee99 --- /dev/null +++ b/docs/cn/Debug_In_Idea.md @@ -0,0 +1,55 @@ +## 本地调试RocketMQ + +### Step0: 解决依赖问题 +1. 运行前下载RocketMQ需要的maven依赖，可以使用`mvn clean install -Dmaven.test.skip=true` +2. 确保本地能够编译通过 + +### Step1: 启动NameServer +1. NamerServer的启动类在`org.apache.rocketmq.namesrv.NamesrvStartup` +2. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=` +![Idea_config_nameserver.png](image/Idea_config_nameserver.png) +3. 运行NameServer，观察到如下日志输出则启动成功 +```shell +The Name Server boot success. serializeType=JSON, address 0.0.0.0:9876 +``` + +### Step2: 启动Broker +1. Broker的启动类在`org.apache.rocketmq.broker.BrokerStartup` +2. 创建`/rocketmq/conf/broker.conf`文件或直接在官方release发布包中拷贝即可 +```shell +# broker.conf + +brokerClusterName = DefaultCluster +brokerName = broker-a +brokerId = 0 +deleteWhen = 04 +fileReservedTime = 48 +brokerRole = ASYNC_MASTER +flushDiskType = ASYNC_FLUSH +namesrvAddr = 127.0.0.1:9876 # name server地址 +``` +3. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=` 以及环境变量`-c /Users/xxx/rocketmq/conf/broker.conf` +![Idea_config_broker.png](image/Idea_config_broker.png) +4. 运行Broker，观察到如下日志则启动成功 +```shell +The broker[broker-a,192.169.1.2:10911] boot success... +``` + +### Step3: 发送或消费消息 +至此已经完成了RocketMQ的启动，可以使用`/example`里的示例进行收发消息 + +### 补充：本地启动Proxy +1. RocketMQ5.x支持了Proxy模式，使用`LOCAL`模式可以免去`Step2`，启动类在`org.apache.rocketmq.proxy.ProxyStartup` +2. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=` +3. 在`/conf/`下新建配置文件`rmq-proxy.json` +```json +{ + "rocketMQClusterName": "DefaultCluster", + "nameSrvAddr": "127.0.0.1:9876", + "proxyMode": "local" +} +``` +4. 运行Proxy，观察到如下日志则启动成功 +```shell +Sat Aug 26 15:29:33 CST 2023 rocketmq-proxy startup successfully +``` \ No newline at end of file diff --git a/docs/cn/image/Idea_config_broker.png b/docs/cn/image/Idea_config_broker.png new file mode 100644 index 00000000000..6fbedcfb627 Binary files /dev/null and b/docs/cn/image/Idea_config_broker.png differ diff --git a/docs/cn/image/Idea_config_nameserver.png b/docs/cn/image/Idea_config_nameserver.png new file mode 100644 index 00000000000..65edd991135 Binary files /dev/null and b/docs/cn/image/Idea_config_nameserver.png differ diff --git a/docs/en/Debug_In_Idea.md b/docs/en/Debug_In_Idea.md new file mode 100644 index 00000000000..9967980671f --- /dev/null +++ b/docs/en/Debug_In_Idea.md @@ -0,0 +1,55 @@ +## How to Debug RocketMQ in Idea + +### Step0: Resolve dependencies +1. To download the Maven dependencies required for running RocketMQ, you can use the following command:`mvn clean install -Dmaven.test.skip=true` +2. Ensure successful local compilation. + +### Step1: Start NameServer +1. The startup class for NameServer is located in `org.apache.rocketmq.namesrv.NamesrvStartup`. +2. Add runtime `ROCKETMQ_HOME=` parameters in `Idea-Edit Configurations`. +![Idea_config_nameserver.png](../cn/image/Idea_config_nameserver.png) +3. Run NameServer and if the following log output is observed, it indicates successful startup. +```shell +The Name Server boot success. serializeType=JSON, address 0.0.0.0:9876 +``` + +### Step2: Start Broker +1. The startup class for Broker is located in`org.apache.rocketmq.broker.BrokerStartup` +2. Create the `/rocketmq/conf/broker.conf` file or simply copy it from the official release package. +```shell +# broker.conf + +brokerClusterName = DefaultCluster +brokerName = broker-a +brokerId = 0 +deleteWhen = 04 +fileReservedTime = 48 +brokerRole = ASYNC_MASTER +flushDiskType = ASYNC_FLUSH +namesrvAddr = 127.0.0.1:9876 # name server地址 +``` +3. Add the runtime parameter `ROCKETMQ_HOME=` and the environment variable `-c /Users/xxx/rocketmq/conf/broker.conf` in `Idea-Edit Configurations`. +![Idea_config_broker.png](../cn/image/Idea_config_broker.png) +4. Run the Broker and if the following log is observed, it indicates successful startup. +```shell +The broker[broker-a,192.169.1.2:10911] boot success... +``` + +### Step3: Send or Consume Messages +RocketMQ startup is now complete. You can use the examples provided in `/example` to send and consume messages. + +### Additional: Start the Proxy locally. +1. RocketMQ 5.x introduced the Proxy mode. Using the `LOCAL` mode eliminates the need for `Step2`. The startup class is located at `org.apache.rocketmq.proxy.ProxyStartup`. +2. Add the runtime parameter `ROCKETMQ_HOME=` in `Idea-Edit Configurations`. +3. Create a new configuration file named `rmq-proxy.json` in the `/conf/` directory. +```json +{ + "rocketMQClusterName": "DefaultCluster", + "nameSrvAddr": "127.0.0.1:9876", + "proxyMode": "local" +} +``` +4. Run the Proxy, and if the following log is observed, it indicates successful startup. +```shell +Sat Aug 26 15:29:33 CST 2023 rocketmq-proxy startup successfully +``` \ No newline at end of file diff --git a/example/pom.xml b/example/pom.xml index 862fc316919..a8c7f538224 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/filter/pom.xml b/filter/pom.xml index 3fe51ceae72..892f46e9d16 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 684b2683c38..e320ed5732f 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java index 15c65ebec9d..be327cffa51 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java @@ -20,10 +20,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RunnableFuture; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.rocketmq.common.ThreadFactoryImpl; @@ -31,6 +28,7 @@ import org.apache.rocketmq.common.future.FutureTaskExt; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.common.utils.NetworkUtil; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.namesrv.kvconfig.KVConfigManager; @@ -62,10 +60,10 @@ public class NamesrvController { private final NettyServerConfig nettyServerConfig; private final NettyClientConfig nettyClientConfig; - private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, + private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, new BasicThreadFactory.Builder().namingPattern("NSScheduledThread").daemon(true).build()); - private final ScheduledExecutorService scanExecutorService = new ScheduledThreadPoolExecutor(1, + private final ScheduledExecutorService scanExecutorService = ThreadUtils.newScheduledThreadPool(1, new BasicThreadFactory.Builder().namingPattern("NSScanScheduledThread").daemon(true).build()); private final KVConfigManager kvConfigManager; @@ -138,20 +136,10 @@ private void initiateNetworkComponents() { private void initiateThreadExecutors() { this.defaultThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getDefaultThreadPoolQueueCapacity()); - this.defaultExecutor = new ThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")) { - @Override - protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { - return new FutureTaskExt<>(runnable, value); - } - }; + this.defaultExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")); this.clientRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getClientRequestThreadPoolQueueCapacity()); - this.clientRequestExecutor = new ThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")) { - @Override - protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { - return new FutureTaskExt<>(runnable, value); - } - }; + this.clientRequestExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")); } private void initiateSslContext() { diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index fada0efd774..485b95c42d7 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -41,7 +41,6 @@ import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; import org.apache.rocketmq.remoting.protocol.body.GetBrokerMemberGroupResponseBody; -import org.apache.rocketmq.remoting.protocol.body.GetRemoteClientConfigBody; import org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody; import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.remoting.protocol.body.TopicList; @@ -132,8 +131,6 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, return this.updateConfig(ctx, request); case RequestCode.GET_NAMESRV_CONFIG: return this.getConfig(ctx, request); - case RequestCode.GET_CLIENT_CONFIG: - return this.getClientConfigs(ctx, request); default: String error = " request type " + request.getCode() + " not supported"; return RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); @@ -661,25 +658,4 @@ private RemotingCommand getConfig(ChannelHandlerContext ctx, RemotingCommand req return response; } - private RemotingCommand getClientConfigs(ChannelHandlerContext ctx, RemotingCommand request) { - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final GetRemoteClientConfigBody body = GetRemoteClientConfigBody.decode(request.getBody(), GetRemoteClientConfigBody.class); - - String content = this.namesrvController.getConfiguration().getClientConfigsFormatString(body.getKeys()); - if (StringUtils.isNotBlank(content)) { - try { - response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); - } catch (UnsupportedEncodingException e) { - log.error("getConfig error, ", e); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("UnsupportedEncodingException " + e); - return response; - } - } - - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); - return response; - } - } diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index aaa4c896cd8..f10c8af6f0e 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 9f0b3eb96ba..a3f7c227050 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 2012 org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -137,7 +137,7 @@ 1.29.0-alpha 2.0.6 2.20.29 - 1.0.3 + 1.0.2 2.13.4.2 @@ -713,7 +713,7 @@ ${slf4j-api.version} - io.github.aliyunmq + org.apache.rocketmq rocketmq-rocksdb ${rocksdb.version} @@ -974,6 +974,11 @@ opentelemetry-sdk ${opentelemetry.version} + + io.opentelemetry + opentelemetry-exporter-logging-otlp + ${opentelemetry.version} + org.slf4j jul-to-slf4j diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel index b4f3c16e22d..cb7af925499 100644 --- a/proxy/BUILD.bazel +++ b/proxy/BUILD.bazel @@ -52,6 +52,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", "@maven//:io_opentelemetry_opentelemetry_exporter_logging", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:io_opentelemetry_opentelemetry_sdk", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", diff --git a/proxy/pom.xml b/proxy/pom.xml index 3fbea107abe..5c5349a8c13 100644 --- a/proxy/pom.xml +++ b/proxy/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java index 06d5f4525f0..3b2ca99bfd0 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java @@ -85,6 +85,7 @@ public static void main(String[] args) { .addService(ChannelzService.newInstance(100)) .addService(ProtoReflectionService.newInstance()) .configInterceptor(accessValidators) + .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS) .build(); PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java index 39caaa0d91d..c0d00d86409 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java @@ -87,6 +87,7 @@ public class ProxyConfig implements ConfigFile { */ private String proxyMode = ProxyMode.CLUSTER.name(); private Integer grpcServerPort = 8081; + private long grpcShutdownTimeSeconds = 30; private int grpcBossLoopNum = 1; private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2; private boolean enableGrpcEpoll = false; @@ -155,14 +156,17 @@ public class ProxyConfig implements ConfigFile { private int consumerProcessorThreadPoolQueueCapacity = 10000; private boolean useEndpointPortFromRequest = false; - private int topicRouteServiceCacheExpiredInSeconds = 20; + + private int topicRouteServiceCacheExpiredSeconds = 300; + private int topicRouteServiceCacheRefreshSeconds = 20; private int topicRouteServiceCacheMaxNum = 20000; private int topicRouteServiceThreadPoolNums = PROCESSOR_NUMBER; private int topicRouteServiceThreadPoolQueueCapacity = 5000; - - private int topicConfigCacheExpiredInSeconds = 20; + private int topicConfigCacheExpiredSeconds = 300; + private int topicConfigCacheRefreshSeconds = 20; private int topicConfigCacheMaxNum = 20000; - private int subscriptionGroupConfigCacheExpiredInSeconds = 20; + private int subscriptionGroupConfigCacheExpiredSeconds = 300; + private int subscriptionGroupConfigCacheRefreshSeconds = 20; private int subscriptionGroupConfigCacheMaxNum = 20000; private int metadataThreadPoolNums = 3; private int metadataThreadPoolQueueCapacity = 100000; @@ -229,6 +233,12 @@ public class ProxyConfig implements ConfigFile { private String remotingAccessAddr = ""; private int remotingListenPort = 8080; + // related to proxy's send strategy in cluster mode. + private boolean sendLatencyEnable = false; + private boolean startDetectorEnable = false; + private int detectTimeout = 200; + private int detectInterval = 2 * 1000; + private int remotingHeartbeatThreadPoolNums = 2 * PROCESSOR_NUMBER; private int remotingTopicRouteThreadPoolNums = 2 * PROCESSOR_NUMBER; private int remotingSendMessageThreadPoolNums = 4 * PROCESSOR_NUMBER; @@ -250,6 +260,8 @@ public class ProxyConfig implements ConfigFile { private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000; private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000; + private boolean enableBatchAck = false; + @Override public void initData() { parseDelayLevel(); @@ -432,6 +444,14 @@ public void setGrpcServerPort(Integer grpcServerPort) { this.grpcServerPort = grpcServerPort; } + public long getGrpcShutdownTimeSeconds() { + return grpcShutdownTimeSeconds; + } + + public void setGrpcShutdownTimeSeconds(long grpcShutdownTimeSeconds) { + this.grpcShutdownTimeSeconds = grpcShutdownTimeSeconds; + } + public boolean isUseEndpointPortFromRequest() { return useEndpointPortFromRequest; } @@ -792,12 +812,20 @@ public void setConsumerProcessorThreadPoolQueueCapacity(int consumerProcessorThr this.consumerProcessorThreadPoolQueueCapacity = consumerProcessorThreadPoolQueueCapacity; } - public int getTopicRouteServiceCacheExpiredInSeconds() { - return topicRouteServiceCacheExpiredInSeconds; + public int getTopicRouteServiceCacheExpiredSeconds() { + return topicRouteServiceCacheExpiredSeconds; + } + + public void setTopicRouteServiceCacheExpiredSeconds(int topicRouteServiceCacheExpiredSeconds) { + this.topicRouteServiceCacheExpiredSeconds = topicRouteServiceCacheExpiredSeconds; + } + + public int getTopicRouteServiceCacheRefreshSeconds() { + return topicRouteServiceCacheRefreshSeconds; } - public void setTopicRouteServiceCacheExpiredInSeconds(int topicRouteServiceCacheExpiredInSeconds) { - this.topicRouteServiceCacheExpiredInSeconds = topicRouteServiceCacheExpiredInSeconds; + public void setTopicRouteServiceCacheRefreshSeconds(int topicRouteServiceCacheRefreshSeconds) { + this.topicRouteServiceCacheRefreshSeconds = topicRouteServiceCacheRefreshSeconds; } public int getTopicRouteServiceCacheMaxNum() { @@ -824,12 +852,20 @@ public void setTopicRouteServiceThreadPoolQueueCapacity(int topicRouteServiceThr this.topicRouteServiceThreadPoolQueueCapacity = topicRouteServiceThreadPoolQueueCapacity; } - public int getTopicConfigCacheExpiredInSeconds() { - return topicConfigCacheExpiredInSeconds; + public int getTopicConfigCacheRefreshSeconds() { + return topicConfigCacheRefreshSeconds; + } + + public void setTopicConfigCacheRefreshSeconds(int topicConfigCacheRefreshSeconds) { + this.topicConfigCacheRefreshSeconds = topicConfigCacheRefreshSeconds; } - public void setTopicConfigCacheExpiredInSeconds(int topicConfigCacheExpiredInSeconds) { - this.topicConfigCacheExpiredInSeconds = topicConfigCacheExpiredInSeconds; + public int getTopicConfigCacheExpiredSeconds() { + return topicConfigCacheExpiredSeconds; + } + + public void setTopicConfigCacheExpiredSeconds(int topicConfigCacheExpiredSeconds) { + this.topicConfigCacheExpiredSeconds = topicConfigCacheExpiredSeconds; } public int getTopicConfigCacheMaxNum() { @@ -840,12 +876,20 @@ public void setTopicConfigCacheMaxNum(int topicConfigCacheMaxNum) { this.topicConfigCacheMaxNum = topicConfigCacheMaxNum; } - public int getSubscriptionGroupConfigCacheExpiredInSeconds() { - return subscriptionGroupConfigCacheExpiredInSeconds; + public int getSubscriptionGroupConfigCacheRefreshSeconds() { + return subscriptionGroupConfigCacheRefreshSeconds; } - public void setSubscriptionGroupConfigCacheExpiredInSeconds(int subscriptionGroupConfigCacheExpiredInSeconds) { - this.subscriptionGroupConfigCacheExpiredInSeconds = subscriptionGroupConfigCacheExpiredInSeconds; + public void setSubscriptionGroupConfigCacheRefreshSeconds(int subscriptionGroupConfigCacheRefreshSeconds) { + this.subscriptionGroupConfigCacheRefreshSeconds = subscriptionGroupConfigCacheRefreshSeconds; + } + + public int getSubscriptionGroupConfigCacheExpiredSeconds() { + return subscriptionGroupConfigCacheExpiredSeconds; + } + + public void setSubscriptionGroupConfigCacheExpiredSeconds(int subscriptionGroupConfigCacheExpiredSeconds) { + this.subscriptionGroupConfigCacheExpiredSeconds = subscriptionGroupConfigCacheExpiredSeconds; } public int getSubscriptionGroupConfigCacheMaxNum() { @@ -1379,4 +1423,52 @@ public long getRemotingWaitTimeMillsInDefaultQueue() { public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) { this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; } + + public boolean isSendLatencyEnable() { + return sendLatencyEnable; + } + + public boolean isStartDetectorEnable() { + return startDetectorEnable; + } + + public void setStartDetectorEnable(boolean startDetectorEnable) { + this.startDetectorEnable = startDetectorEnable; + } + + public void setSendLatencyEnable(boolean sendLatencyEnable) { + this.sendLatencyEnable = sendLatencyEnable; + } + + public boolean getStartDetectorEnable() { + return this.startDetectorEnable; + } + + public boolean getSendLatencyEnable() { + return this.sendLatencyEnable; + } + + public int getDetectTimeout() { + return detectTimeout; + } + + public void setDetectTimeout(int detectTimeout) { + this.detectTimeout = detectTimeout; + } + + public int getDetectInterval() { + return detectInterval; + } + + public void setDetectInterval(int detectInterval) { + this.detectInterval = detectInterval; + } + + public boolean isEnableBatchAck() { + return enableBatchAck; + } + + public void setEnableBatchAck(boolean enableBatchAck) { + this.enableBatchAck = enableBatchAck; + } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java index 1bffa3c0be1..d5b896fe144 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java @@ -29,8 +29,14 @@ public class GrpcServer implements StartAndShutdown { private final Server server; - protected GrpcServer(Server server) { + private final long timeout; + + private final TimeUnit unit; + + protected GrpcServer(Server server, long timeout, TimeUnit unit) { this.server = server; + this.timeout = timeout; + this.unit = unit; } public void start() throws Exception { @@ -40,7 +46,7 @@ public void start() throws Exception { public void shutdown() { try { - this.server.shutdown().awaitTermination(30, TimeUnit.SECONDS); + this.server.shutdown().awaitTermination(timeout, unit); log.info("grpc server shutdown successfully."); } catch (Exception e) { e.printStackTrace(); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java index 9cddd301373..0e79006f6b3 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java @@ -41,6 +41,10 @@ public class GrpcServerBuilder { private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); protected NettyServerBuilder serverBuilder; + protected long time = 30; + + protected TimeUnit unit = TimeUnit.SECONDS; + public static GrpcServerBuilder newBuilder(ThreadPoolExecutor executor, int port) { return new GrpcServerBuilder(executor, port); } @@ -77,6 +81,12 @@ protected GrpcServerBuilder(ThreadPoolExecutor executor, int port) { port, bossLoopNum, workerLoopNum, maxInboundMessageSize); } + public GrpcServerBuilder shutdownTime(long time, TimeUnit unit) { + this.time = time; + this.unit = unit; + return this; + } + public GrpcServerBuilder addService(BindableService service) { this.serverBuilder.addService(service); return this; @@ -93,7 +103,7 @@ public GrpcServerBuilder appendInterceptor(ServerInterceptor interceptor) { } public GrpcServer build() { - return new GrpcServer(this.serverBuilder.build()); + return new GrpcServer(this.serverBuilder.build(), time, unit); } public GrpcServerBuilder configInterceptor(List accessValidators) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java index ee167bd7bee..b584ddfbdc6 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java @@ -24,6 +24,7 @@ import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiators; import io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiationEvent; import io.grpc.netty.shaded.io.netty.buffer.ByteBuf; +import io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil; import io.grpc.netty.shaded.io.netty.channel.ChannelHandler; import io.grpc.netty.shaded.io.netty.channel.ChannelHandlerContext; import io.grpc.netty.shaded.io.netty.channel.ChannelInboundHandlerAdapter; @@ -44,6 +45,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.constant.HAProxyConstants; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.BinaryUtil; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.proxy.config.ConfigurationManager; @@ -191,9 +193,13 @@ private void handleWithMessage(HAProxyMessage msg) { } if (CollectionUtils.isNotEmpty(msg.tlvs())) { msg.tlvs().forEach(tlv -> { + byte[] valueBytes = ByteBufUtil.getBytes(tlv.content()); + if (!BinaryUtil.isAscii(valueBytes)) { + return; + } Attributes.Key key = AttributeKeys.valueOf( HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); - String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8)); + String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8)); builder.set(key, value); }); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java index 14330dd8d48..a18cf7600c1 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java @@ -21,13 +21,13 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.ThreadFactoryImpl; -import org.apache.rocketmq.proxy.common.ProxyContext; import org.apache.rocketmq.common.utils.StartAndShutdown; +import org.apache.rocketmq.common.utils.ThreadUtils; +import org.apache.rocketmq.proxy.common.ProxyContext; import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.config.ProxyConfig; import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; @@ -43,7 +43,7 @@ public class GrpcChannelManager implements StartAndShutdown { protected final AtomicLong nonceIdGenerator = new AtomicLong(0); protected final ConcurrentMap resultNonceFutureMap = new ConcurrentHashMap<>(); - protected final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + protected final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( new ThreadFactoryImpl("GrpcChannelManager_") ); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java index 9a3a772017e..97c716c8ff3 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java @@ -31,12 +31,15 @@ import org.apache.rocketmq.common.consumer.ReceiptHandle; import org.apache.rocketmq.proxy.common.MessageReceiptHandle; import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.BatchAckResult; import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; public class AckMessageActivity extends AbstractMessingActivity { @@ -50,60 +53,98 @@ public CompletableFuture ackMessage(ProxyContext ctx, AckMes try { validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); - - CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; - for (int i = 0; i < request.getEntriesCount(); i++) { - futures[i] = processAckMessage(ctx, request, request.getEntries(i)); + String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); + String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()); + if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) { + future = ackMessageInBatch(ctx, group, topic, request); + } else { + future = ackMessageOneByOne(ctx, group, topic, request); } - CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { - if (throwable != null) { - future.completeExceptionally(throwable); - return; - } + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + protected CompletableFuture ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) { + List handleMessageList = new ArrayList<>(request.getEntriesCount()); + for (AckMessageEntry ackMessageEntry : request.getEntriesList()) { + String handleString = getHandleString(ctx, group, request, ackMessageEntry); + handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId())); + } + return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic) + .thenApply(batchAckResultList -> { + AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder(); Set responseCodes = new HashSet<>(); - List entryList = new ArrayList<>(); - for (CompletableFuture entryFuture : futures) { - AckMessageResultEntry entryResult = entryFuture.join(); - responseCodes.add(entryResult.getStatus().getCode()); - entryList.add(entryResult); - } - AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() - .addAllEntries(entryList); - if (responseCodes.size() > 1) { - responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); - } else if (responseCodes.size() == 1) { - Code code = responseCodes.stream().findAny().get(); - responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); - } else { - responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); + for (BatchAckResult batchAckResult : batchAckResultList) { + AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult); + responseBuilder.addEntries(entry); + responseCodes.add(entry.getStatus().getCode()); } - future.complete(responseBuilder.build()); + setAckResponseStatus(responseBuilder, responseCodes); + return responseBuilder.build(); }); - } catch (Throwable t) { - future.completeExceptionally(t); + } + + protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) { + ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage(); + AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder() + .setMessageId(handleMessage.getMessageId()) + .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle()); + if (batchAckResult.getProxyException() != null) { + resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException())); + } else { + AckResult ackResult = batchAckResult.getAckResult(); + if (AckStatus.OK.equals(ackResult.getStatus())) { + resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); + } else { + resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")); + } } - return future; + return resultBuilder.build(); } - protected CompletableFuture processAckMessage(ProxyContext ctx, AckMessageRequest request, + protected CompletableFuture ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) { + CompletableFuture resultFuture = new CompletableFuture<>(); + CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; + for (int i = 0; i < request.getEntriesCount(); i++) { + futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i)); + } + CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { + if (throwable != null) { + resultFuture.completeExceptionally(throwable); + return; + } + + Set responseCodes = new HashSet<>(); + List entryList = new ArrayList<>(); + for (CompletableFuture entryFuture : futures) { + AckMessageResultEntry entryResult = entryFuture.join(); + responseCodes.add(entryResult.getStatus().getCode()); + entryList.add(entryResult); + } + AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() + .addAllEntries(entryList); + setAckResponseStatus(responseBuilder, responseCodes); + resultFuture.complete(responseBuilder.build()); + }); + return resultFuture; + } + + protected CompletableFuture processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request, AckMessageEntry ackMessageEntry) { CompletableFuture future = new CompletableFuture<>(); try { - String handleString = ackMessageEntry.getReceiptHandle(); - - String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); - MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); - if (messageReceiptHandle != null) { - handleString = messageReceiptHandle.getReceiptHandleStr(); - } + String handleString = this.getHandleString(ctx, group, request, ackMessageEntry); CompletableFuture ackResultFuture = this.messagingProcessor.ackMessage( ctx, ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId(), group, - GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); + topic + ); ackResultFuture.thenAccept(result -> { future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result)); }).exceptionally(t -> { @@ -139,4 +180,25 @@ protected AckMessageResultEntry convertToAckMessageResultEntry(ProxyContext ctx, .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")) .build(); } + + protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set responseCodes) { + if (responseCodes.size() > 1) { + responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); + } else if (responseCodes.size() == 1) { + Code code = responseCodes.stream().findAny().get(); + responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); + } else { + responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); + } + } + + protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) { + String handleString = ackMessageEntry.getReceiptHandle(); + + MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); + if (messageReceiptHandle != null) { + handleString = messageReceiptHandle.getReceiptHandleStr(); + } + return handleString; + } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java index 6146c80cd0f..f670df2050e 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java @@ -382,7 +382,7 @@ public AddressableMessageQueue select(ProxyContext ctx, MessageQueueView message int bucket = Hashing.consistentHash(shardingKey.hashCode(), writeQueues.size()); targetMessageQueue = writeQueues.get(bucket); } else { - targetMessageQueue = messageQueueView.getWriteSelector().selectOne(false); + targetMessageQueue = messageQueueView.getWriteSelector().selectOneByPipeline(false); } return targetMessageQueue; } catch (Exception e) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java index f5050858f61..2b8dac5d8be 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java @@ -21,15 +21,16 @@ import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.ObservableLongGauge; +import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; -import io.opentelemetry.exporter.logging.LoggingMetricExporter; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.export.MetricExporter; import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; import io.opentelemetry.sdk.resources.Resource; import java.util.HashMap; @@ -42,9 +43,9 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.metrics.MetricsExporterType; import org.apache.rocketmq.common.utils.StartAndShutdown; -import org.apache.rocketmq.proxy.config.ProxyConfig; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.proxy.config.ProxyConfig; import org.slf4j.bridge.SLF4JBridgeHandler; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.AGGREGATION_DELTA; @@ -67,7 +68,7 @@ public class ProxyMetricsManager implements StartAndShutdown { private OtlpGrpcMetricExporter metricExporter; private PeriodicMetricReader periodicMetricReader; private PrometheusHttpServer prometheusHttpServer; - private LoggingMetricExporter loggingMetricExporter; + private MetricExporter loggingMetricExporter; public static ObservableLongGauge proxyUp = null; @@ -221,8 +222,8 @@ public void start() throws Exception { if (metricsExporterType == MetricsExporterType.LOG) { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); - loggingMetricExporter = LoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); - java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); + java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) .setInterval(proxyConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) .build(); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java index b61c3df9e52..c63212c2314 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java @@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { protected MessagingProcessor messagingProcessor; protected ServiceManager serviceManager; + protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); + public AbstractProcessor(MessagingProcessor messagingProcessor, ServiceManager serviceManager) { this.messagingProcessor = messagingProcessor; @@ -35,7 +37,7 @@ public AbstractProcessor(MessagingProcessor messagingProcessor, protected void validateReceiptHandle(ReceiptHandle handle) { if (handle.isExpired()) { - throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); + throw EXPIRED_HANDLE_PROXY_EXCEPTION; } } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java new file mode 100644 index 00000000000..dfb9c9b9e02 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor; + +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; + +public class BatchAckResult { + + private final ReceiptHandleMessage receiptHandleMessage; + private AckResult ackResult; + private ProxyException proxyException; + + public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, + AckResult ackResult) { + this.receiptHandleMessage = receiptHandleMessage; + this.ackResult = ackResult; + } + + public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, + ProxyException proxyException) { + this.receiptHandleMessage = receiptHandleMessage; + this.proxyException = proxyException; + } + + public ReceiptHandleMessage getReceiptHandleMessage() { + return receiptHandleMessage; + } + + public AckResult getAckResult() { + return ackResult; + } + + public ProxyException getProxyException() { + return proxyException; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java index 656a6339dc3..f3522b37401 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.utils.FutureUtils; import org.apache.rocketmq.proxy.common.utils.ProxyUtils; import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; @@ -241,6 +242,69 @@ public CompletableFuture ackMessage( return FutureUtils.addExecutor(future, this.executor); } + public CompletableFuture> batchAckMessage( + ProxyContext ctx, + List handleMessageList, + String consumerGroup, + String topic, + long timeoutMillis + ) { + CompletableFuture> future = new CompletableFuture<>(); + try { + List batchAckResultList = new ArrayList<>(handleMessageList.size()); + Map> brokerHandleListMap = new HashMap<>(); + + for (ReceiptHandleMessage handleMessage : handleMessageList) { + if (handleMessage.getReceiptHandle().isExpired()) { + batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION)); + continue; + } + List brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>()); + brokerHandleList.add(handleMessage); + } + + if (brokerHandleListMap.isEmpty()) { + return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor); + } + Set>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet(); + CompletableFuture>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()]; + int futureIndex = 0; + for (Map.Entry> entry : brokerHandleListMapEntrySet) { + futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis); + } + CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { + if (throwable != null) { + future.completeExceptionally(throwable); + } + for (CompletableFuture> resultFuture : futures) { + batchAckResultList.addAll(resultFuture.join()); + } + future.complete(batchAckResultList); + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List handleMessageList, long timeoutMillis) { + return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis) + .thenApply(result -> { + List results = new ArrayList<>(); + for (ReceiptHandleMessage handleMessage : handleMessageList) { + results.add(new BatchAckResult(handleMessage, result)); + } + return results; + }) + .exceptionally(throwable -> { + List results = new ArrayList<>(); + for (ReceiptHandleMessage handleMessage : handleMessageList) { + results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable))); + } + return results; + }); + } + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { CompletableFuture future = new CompletableFuture<>(); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java index 188cb7b9bd3..ba150051bc0 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java @@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ProxyConfig; import org.apache.rocketmq.proxy.service.ServiceManager; import org.apache.rocketmq.proxy.service.ServiceManagerFactory; +import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; import org.apache.rocketmq.proxy.service.metadata.MetadataService; import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; @@ -183,6 +184,12 @@ public CompletableFuture ackMessage(ProxyContext ctx, ReceiptHandle h return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis); } + @Override + public CompletableFuture> batchAckMessage(ProxyContext ctx, + List handleMessageList, String consumerGroup, String topic, long timeoutMillis) { + return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis); + } + @Override public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java index d86be0bd887..2ae7418ba72 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java @@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; import org.apache.rocketmq.proxy.common.ProxyContext; import org.apache.rocketmq.common.utils.StartAndShutdown; +import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; import org.apache.rocketmq.proxy.service.metadata.MetadataService; import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; @@ -155,6 +156,23 @@ CompletableFuture ackMessage( long timeoutMillis ); + default CompletableFuture> batchAckMessage( + ProxyContext ctx, + List handleMessageList, + String consumerGroup, + String topic + ) { + return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS); + } + + CompletableFuture> batchAckMessage( + ProxyContext ctx, + List handleMessageList, + String consumerGroup, + String topic, + long timeoutMillis + ); + default CompletableFuture changeInvisibleTime( ProxyContext ctx, ReceiptHandle handle, diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java index 0d0c6216862..a80f6df0b07 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; + import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; @@ -66,6 +67,8 @@ public ProducerProcessor(MessagingProcessor messagingProcessor, public CompletableFuture> sendMessage(ProxyContext ctx, QueueSelector queueSelector, String producerGroup, int sysFlag, List messageList, long timeoutMillis) { CompletableFuture> future = new CompletableFuture<>(); + long beginTimestampFirst = System.currentTimeMillis(); + AddressableMessageQueue messageQueue = null; try { Message message = messageList.get(0); String topic = message.getTopic(); @@ -79,7 +82,7 @@ public CompletableFuture> sendMessage(ProxyContext ctx, QueueSe } } } - AddressableMessageQueue messageQueue = queueSelector.select(ctx, + messageQueue = queueSelector.select(ctx, this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(ctx, topic)); if (messageQueue == null) { throw new ProxyException(ProxyExceptionCode.FORBIDDEN, "no writable queue"); @@ -90,6 +93,7 @@ public CompletableFuture> sendMessage(ProxyContext ctx, QueueSe } SendMessageRequestHeader requestHeader = buildSendMessageRequestHeader(messageList, producerGroup, sysFlag, messageQueue.getQueueId()); + AddressableMessageQueue finalMessageQueue = messageQueue; future = this.serviceManager.getMessageService().sendMessage( ctx, messageQueue, @@ -102,11 +106,19 @@ public CompletableFuture> sendMessage(ProxyContext ctx, QueueSe if (SendStatus.SEND_OK.equals(sendResult.getSendStatus()) && tranType == MessageSysFlag.TRANSACTION_PREPARED_TYPE && StringUtils.isNotBlank(sendResult.getTransactionId())) { - fillTransactionData(ctx, producerGroup, messageQueue, sendResult, messageList); + fillTransactionData(ctx, producerGroup, finalMessageQueue, sendResult, messageList); } } return sendResultList; - }, this.executor); + }, this.executor) + .whenComplete((result, exception) -> { + long endTimestamp = System.currentTimeMillis(); + if (exception != null) { + this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(), endTimestamp - beginTimestampFirst, true, false); + } else { + this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(),endTimestamp - beginTimestampFirst, false, true); + } + }); } catch (Throwable t) { future.completeExceptionally(t); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java index bcc9edd091c..3227d1e1c63 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java @@ -22,17 +22,16 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.acl.AccessValidator; -import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.future.FutureTaskExt; import org.apache.rocketmq.common.thread.ThreadPoolMonitor; import org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor; import org.apache.rocketmq.common.utils.StartAndShutdown; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.proxy.config.ConfigurationManager; @@ -51,10 +50,12 @@ import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline; import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline; import org.apache.rocketmq.remoting.ChannelEventListener; +import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RemotingServer; import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.RequestTask; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.netty.TlsSystemConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RequestCode; @@ -178,7 +179,7 @@ public RemotingProtocolServer(MessagingProcessor messagingProcessor, List invokeToClient(Channel channel, Remoti long timeoutMillis) { CompletableFuture future = new CompletableFuture<>(); try { - this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, responseFuture -> { - if (responseFuture.getResponseCommand() == null) { - future.completeExceptionally(new MQClientException("response is null after send request to client", responseFuture.getCause())); - return; + this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + future.complete(response); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } - future.complete(responseFuture.getResponseCommand()); }); } catch (Throwable t) { future.completeExceptionally(t); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java index 40946cabf86..d755fdcc493 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java @@ -34,6 +34,7 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.proxy.common.channel.ChannelHelper; +import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; import org.apache.rocketmq.proxy.common.utils.FutureUtils; import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter; @@ -158,10 +159,15 @@ protected CompletableFuture processGetConsumerRunningInfo(RemotingCommand if (response.getCode() == ResponseCode.SUCCESS) { ConsumerRunningInfo consumerRunningInfo = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class); responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", consumerRunningInfo)); + } else { + String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark()); + RuntimeException e = new RuntimeException(errMsg); + responseFuture.completeExceptionally(e); } - String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark()); - RuntimeException e = new RuntimeException(errMsg); - responseFuture.completeExceptionally(e); + }) + .exceptionally(t -> { + responseFuture.completeExceptionally(ExceptionUtils.getRealException(t)); + return null; }); return CompletableFuture.completedFuture(null); } catch (Throwable t) { @@ -183,10 +189,15 @@ protected CompletableFuture processConsumeMessageDirectly(RemotingCommand if (response.getCode() == ResponseCode.SUCCESS) { ConsumeMessageDirectlyResult result = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class); responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", result)); + } else { + String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark()); + RuntimeException e = new RuntimeException(errMsg); + responseFuture.completeExceptionally(e); } - String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark()); - RuntimeException e = new RuntimeException(errMsg); - responseFuture.completeExceptionally(e); + }) + .exceptionally(t -> { + responseFuture.completeExceptionally(ExceptionUtils.getRealException(t)); + return null; }); return CompletableFuture.completedFuture(null); } catch (Throwable t) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java index d2ddfc3527b..9786cec5577 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.proxy.service; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.client.ClientChannelInfo; @@ -27,23 +26,24 @@ import org.apache.rocketmq.broker.client.ProducerGroupEvent; import org.apache.rocketmq.broker.client.ProducerManager; import org.apache.rocketmq.client.common.NameserverAccessConfig; +import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor; +import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; -import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; import org.apache.rocketmq.proxy.common.ProxyContext; import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.config.ProxyConfig; import org.apache.rocketmq.proxy.service.admin.AdminService; import org.apache.rocketmq.proxy.service.admin.DefaultAdminService; import org.apache.rocketmq.proxy.service.client.ClusterConsumerManager; +import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor; import org.apache.rocketmq.proxy.service.message.ClusterMessageService; import org.apache.rocketmq.proxy.service.message.MessageService; import org.apache.rocketmq.proxy.service.metadata.ClusterMetadataService; import org.apache.rocketmq.proxy.service.metadata.MetadataService; -import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor; -import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; -import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor; import org.apache.rocketmq.proxy.service.relay.ClusterProxyRelayService; import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; import org.apache.rocketmq.proxy.service.route.ClusterTopicRouteService; @@ -73,7 +73,7 @@ public ClusterServiceManager(RPCHook rpcHook) { ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); NameserverAccessConfig nameserverAccessConfig = new NameserverAccessConfig(proxyConfig.getNamesrvAddr(), proxyConfig.getNamesrvDomain(), proxyConfig.getNamesrvDomainSubgroup()); - this.scheduledExecutorService = Executors.newScheduledThreadPool(3); + this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(3); this.messagingClientAPIFactory = new MQClientAPIFactory( nameserverAccessConfig, diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java index 4d1ca7b6699..59cd92685a3 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.proxy.service; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; @@ -28,6 +27,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; import org.apache.rocketmq.common.utils.StartAndShutdown; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.config.ProxyConfig; import org.apache.rocketmq.proxy.service.admin.AdminService; @@ -58,7 +58,7 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser private final MQClientAPIFactory mqClientAPIFactory; private final ChannelManager channelManager; - private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( new ThreadFactoryImpl("LocalServiceManagerScheduledThread")); public LocalServiceManager(BrokerController brokerController, RPCHook rpcHook) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java index 9f163f1b987..70b72deae18 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java @@ -20,9 +20,11 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; import org.apache.rocketmq.client.consumer.AckResult; import org.apache.rocketmq.client.consumer.PopResult; import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.consumer.ReceiptHandle; import org.apache.rocketmq.common.message.Message; @@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyException; import org.apache.rocketmq.proxy.common.ProxyExceptionCode; import org.apache.rocketmq.proxy.common.utils.FutureUtils; -import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; import org.apache.rocketmq.proxy.service.route.TopicRouteService; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -137,6 +138,19 @@ public CompletableFuture ackMessage(ProxyContext ctx, ReceiptHandle h ); } + @Override + public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, String consumerGroup, + String topic, long timeoutMillis) { + List extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList()); + return this.mqClientAPIFactory.getClient().batchAckMessageAsync( + this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()), + topic, + consumerGroup, + extraInfoList, + timeoutMillis + ); + } + @Override public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, PullMessageRequestHeader requestHeader, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java index eb2c4d9ee91..ca7dcc9eb0c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java @@ -19,6 +19,7 @@ import io.netty.channel.ChannelHandlerContext; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RequestCode; import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.body.BatchAck; +import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; @@ -364,6 +367,61 @@ public CompletableFuture ackMessage(ProxyContext ctx, ReceiptHandle h }); } + @Override + public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, + String consumerGroup, String topic, long timeoutMillis) { + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); + RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); + + Map batchAckMap = new HashMap<>(); + for (ReceiptHandleMessage receiptHandleMessage : handleList) { + String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle(); + String[] extraInfoData = ExtraInfoUtil.split(extraInfo); + String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + + ExtraInfoUtil.getQueueId(extraInfoData) + "@" + + ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + + ExtraInfoUtil.getPopTime(extraInfoData); + BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { + BatchAck newBatchAck = new BatchAck(); + newBatchAck.setConsumerGroup(consumerGroup); + newBatchAck.setTopic(topic); + newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); + newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); + newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); + newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); + newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); + newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); + newBatchAck.setBitSet(new BitSet()); + return newBatchAck; + }); + bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); + } + BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); + requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); + requestBody.setAcks(new ArrayList<>(batchAckMap.values())); + + command.setBody(requestBody.encode()); + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand response = brokerController.getAckMessageProcessor() + .processRequest(channelHandlerContext, command); + future.complete(response); + } catch (Exception e) { + log.error("Fail to process batchAckMessage command", e); + future.completeExceptionally(e); + } + return future.thenApply(r -> { + AckResult ackResult = new AckResult(); + if (ResponseCode.SUCCESS == r.getCode()) { + ackResult.setStatus(AckStatus.OK); + } else { + ackResult.setStatus(AckStatus.NO_EXIST); + } + return ackResult; + }); + } + @Override public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, PullMessageRequestHeader requestHeader, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java index 15da1715402..58a835adb46 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java @@ -91,6 +91,14 @@ CompletableFuture ackMessage( long timeoutMillis ); + CompletableFuture batchAckMessage( + ProxyContext ctx, + List handleList, + String consumerGroup, + String topic, + long timeoutMillis + ); + CompletableFuture pullMessage( ProxyContext ctx, AddressableMessageQueue messageQueue, diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java similarity index 61% rename from broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java rename to proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java index f132efaebcc..ae63fed491e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java @@ -15,25 +15,25 @@ * limitations under the License. */ -package org.apache.rocketmq.broker.latency; +package org.apache.rocketmq.proxy.service.message; -import java.util.concurrent.Callable; -import java.util.concurrent.FutureTask; +import org.apache.rocketmq.common.consumer.ReceiptHandle; -public class FutureTaskExt extends FutureTask { - private final Runnable runnable; +public class ReceiptHandleMessage { - public FutureTaskExt(final Callable callable) { - super(callable); - this.runnable = null; + private final ReceiptHandle receiptHandle; + private final String messageId; + + public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) { + this.receiptHandle = receiptHandle; + this.messageId = messageId; } - public FutureTaskExt(final Runnable runnable, final V result) { - super(runnable, result); - this.runnable = runnable; + public ReceiptHandle getReceiptHandle() { + return receiptHandle; } - public Runnable getRunnable() { - return runnable; + public String getMessageId() { + return messageId; } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java index bc9582ad816..d34a0efd9e1 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java @@ -69,11 +69,13 @@ public ClusterMetadataService(TopicRouteService topicRouteService, MQClientAPIFa ); this.topicConfigCache = CacheBuilder.newBuilder() .maximumSize(config.getTopicConfigCacheMaxNum()) - .refreshAfterWrite(config.getTopicConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) + .expireAfterAccess(config.getTopicConfigCacheExpiredSeconds(), TimeUnit.SECONDS) + .refreshAfterWrite(config.getTopicConfigCacheRefreshSeconds(), TimeUnit.SECONDS) .build(new ClusterTopicConfigCacheLoader()); this.subscriptionGroupConfigCache = CacheBuilder.newBuilder() .maximumSize(config.getSubscriptionGroupConfigCacheMaxNum()) - .refreshAfterWrite(config.getSubscriptionGroupConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) + .expireAfterAccess(config.getSubscriptionGroupConfigCacheExpiredSeconds(), TimeUnit.SECONDS) + .refreshAfterWrite(config.getSubscriptionGroupConfigCacheRefreshSeconds(), TimeUnit.SECONDS) .build(new ClusterSubscriptionGroupConfigCacheLoader()); this.init(); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java index 69f44344a03..207603fe811 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java @@ -24,7 +24,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -42,20 +41,21 @@ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; import org.apache.rocketmq.common.utils.StartAndShutdown; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; -import org.apache.rocketmq.proxy.common.RenewEvent; import org.apache.rocketmq.proxy.common.MessageReceiptHandle; import org.apache.rocketmq.proxy.common.ProxyContext; import org.apache.rocketmq.proxy.common.ProxyException; import org.apache.rocketmq.proxy.common.ProxyExceptionCode; import org.apache.rocketmq.proxy.common.ReceiptHandleGroup; +import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey; +import org.apache.rocketmq.proxy.common.RenewEvent; import org.apache.rocketmq.proxy.common.RenewStrategyPolicy; import org.apache.rocketmq.proxy.common.channel.ChannelHelper; import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.config.ProxyConfig; -import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey; import org.apache.rocketmq.proxy.service.metadata.MetadataService; import org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; @@ -68,7 +68,7 @@ public class DefaultReceiptHandleManager extends AbstractStartAndShutdown implem protected final StateEventListener eventListener; protected final static RetryPolicy RENEW_POLICY = new RenewStrategyPolicy(); protected final ScheduledExecutorService scheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); + ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); protected final ThreadPoolExecutor renewalWorkerService; public DefaultReceiptHandleManager(MetadataService metadataService, ConsumerManager consumerManager, StateEventListener eventListener) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java index d67b68f38e9..aced15cee51 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java @@ -54,7 +54,7 @@ public LocalTopicRouteService(BrokerController brokerController, MQClientAPIFact @Override public MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topic) throws Exception { TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic); - return new MessageQueueView(topic, toTopicRouteData(topicConfig)); + return new MessageQueueView(topic, toTopicRouteData(topicConfig), null); } @Override diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java index 85cd18d45c9..f25fb907ef2 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.proxy.service.route; import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; import com.google.common.math.IntMath; import java.util.ArrayList; import java.util.Collections; @@ -30,6 +31,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.latency.MQFaultStrategy; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.protocol.route.QueueData; @@ -44,8 +47,9 @@ public class MessageQueueSelector { private final Map brokerNameQueueMap = new ConcurrentHashMap<>(); private final AtomicInteger queueIndex; private final AtomicInteger brokerIndex; + private MQFaultStrategy mqFaultStrategy; - public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read) { + public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, MQFaultStrategy mqFaultStrategy, boolean read) { if (read) { this.queues.addAll(buildRead(topicRouteWrapper)); } else { @@ -55,6 +59,7 @@ public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read) { Random random = new Random(); this.queueIndex = new AtomicInteger(random.nextInt()); this.brokerIndex = new AtomicInteger(random.nextInt()); + this.mqFaultStrategy = mqFaultStrategy; } private static List buildRead(TopicRouteWrapper topicRoute) { @@ -154,6 +159,86 @@ public AddressableMessageQueue selectOne(boolean onlyBroker) { return selectOneByIndex(nextIndex, onlyBroker); } + public AddressableMessageQueue selectOneByPipeline(boolean onlyBroker) { + if (mqFaultStrategy != null && mqFaultStrategy.isSendLatencyFaultEnable()) { + List messageQueueList = null; + MessageQueue messageQueue = null; + if (onlyBroker) { + messageQueueList = transferAddressableQueues(brokerActingQueues); + } else { + messageQueueList = transferAddressableQueues(queues); + } + AddressableMessageQueue addressableMessageQueue = null; + + // use both available filter. + messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, + mqFaultStrategy.getAvailableFilter(), mqFaultStrategy.getReachableFilter()); + addressableMessageQueue = transferQueue2Addressable(messageQueue); + if (addressableMessageQueue != null) { + return addressableMessageQueue; + } + + // use available filter. + messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, + mqFaultStrategy.getAvailableFilter()); + addressableMessageQueue = transferQueue2Addressable(messageQueue); + if (addressableMessageQueue != null) { + return addressableMessageQueue; + } + + // no available filter, then use reachable filter. + messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, + mqFaultStrategy.getReachableFilter()); + addressableMessageQueue = transferQueue2Addressable(messageQueue); + if (addressableMessageQueue != null) { + return addressableMessageQueue; + } + } + + // SendLatency is not enabled, or no queue is selected, then select by index. + return selectOne(onlyBroker); + } + + private MessageQueue selectOneMessageQueue(List messageQueueList, AtomicInteger sendQueue, TopicPublishInfo.QueueFilter...filter) { + if (messageQueueList == null || messageQueueList.isEmpty()) { + return null; + } + if (filter != null && filter.length != 0) { + for (int i = 0; i < messageQueueList.size(); i++) { + int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); + MessageQueue mq = messageQueueList.get(index); + boolean filterResult = true; + for (TopicPublishInfo.QueueFilter f: filter) { + Preconditions.checkNotNull(f); + filterResult &= f.filter(mq); + } + if (filterResult) { + return mq; + } + } + } + return null; + } + + public List transferAddressableQueues(List addressableMessageQueueList) { + if (addressableMessageQueueList == null) { + return null; + } + + return addressableMessageQueueList.stream() + .map(AddressableMessageQueue::getMessageQueue) + .collect(Collectors.toList()); + } + + private AddressableMessageQueue transferQueue2Addressable(MessageQueue messageQueue) { + for (AddressableMessageQueue amq: queues) { + if (amq.getMessageQueue().equals(messageQueue)) { + return amq; + } + } + return null; + } + public AddressableMessageQueue selectNextOne(AddressableMessageQueue last) { boolean onlyBroker = last.getQueueId() < 0; AddressableMessageQueue newOne = last; @@ -190,6 +275,14 @@ public List getBrokerActingQueues() { return brokerActingQueues; } + public MQFaultStrategy getMQFaultStrategy() { + return mqFaultStrategy; + } + + public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { + this.mqFaultStrategy = mqFaultStrategy; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java index fe5387cfd7e..898e529f8cb 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java @@ -17,20 +17,21 @@ package org.apache.rocketmq.proxy.service.route; import com.google.common.base.MoreObjects; +import org.apache.rocketmq.client.latency.MQFaultStrategy; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; public class MessageQueueView { - public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData()); + public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData(), null); private final MessageQueueSelector readSelector; private final MessageQueueSelector writeSelector; private final TopicRouteWrapper topicRouteWrapper; - public MessageQueueView(String topic, TopicRouteData topicRouteData) { + public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) { this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic); - this.readSelector = new MessageQueueSelector(topicRouteWrapper, true); - this.writeSelector = new MessageQueueSelector(topicRouteWrapper, false); + this.readSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, true); + this.writeSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, false); } public TopicRouteData getTopicRouteData() { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java index e012a5465a4..ccf094c03aa 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java @@ -19,19 +19,24 @@ import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.base.Optional; import java.time.Duration; import java.util.List; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; +import org.apache.rocketmq.client.latency.MQFaultStrategy; +import org.apache.rocketmq.client.latency.Resolver; +import org.apache.rocketmq.client.latency.ServiceDetector; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.thread.ThreadPoolMonitor; import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.proxy.common.Address; @@ -39,6 +44,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.config.ProxyConfig; import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -47,6 +53,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); private final MQClientAPIFactory mqClientAPIFactory; + private MQFaultStrategy mqFaultStrategy; protected final LoadingCache topicCache; protected final ScheduledExecutorService scheduledExecutorService; @@ -55,7 +62,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) { ProxyConfig config = ConfigurationManager.getProxyConfig(); - this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( new ThreadFactoryImpl("TopicRouteService_") ); this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor( @@ -68,10 +75,13 @@ public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) { ); this.mqClientAPIFactory = mqClientAPIFactory; - this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()). - refreshAfterWrite(config.getTopicRouteServiceCacheExpiredInSeconds(), TimeUnit.SECONDS). - executor(cacheRefreshExecutor).build(new CacheLoader() { - @Override public @Nullable MessageQueueView load(String topic) throws Exception { + this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()) + .expireAfterAccess(config.getTopicRouteServiceCacheExpiredSeconds(), TimeUnit.SECONDS) + .refreshAfterWrite(config.getTopicRouteServiceCacheRefreshSeconds(), TimeUnit.SECONDS) + .executor(cacheRefreshExecutor) + .build(new CacheLoader() { + @Override + public @Nullable MessageQueueView load(String topic) throws Exception { try { TopicRouteData topicRouteData = mqClientAPIFactory.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis()); return buildMessageQueueView(topic, topicRouteData); @@ -83,7 +93,8 @@ public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) { } } - @Override public @Nullable MessageQueueView reload(@NonNull String key, + @Override + public @Nullable MessageQueueView reload(@NonNull String key, @NonNull MessageQueueView oldValue) throws Exception { try { return load(key); @@ -93,15 +104,91 @@ public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) { } } }); - + ServiceDetector serviceDetector = new ServiceDetector() { + @Override + public boolean detect(String endpoint, long timeoutMillis) { + Optional candidateTopic = pickTopic(); + if (!candidateTopic.isPresent()) { + return false; + } + try { + GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); + requestHeader.setTopic(candidateTopic.get()); + requestHeader.setQueueId(0); + Long maxOffset = mqClientAPIFactory.getClient().getMaxOffset(endpoint, requestHeader, timeoutMillis).get(); + return true; + } catch (Exception e) { + return false; + } + } + }; + mqFaultStrategy = new MQFaultStrategy(extractClientConfigFromProxyConfig(config), new Resolver() { + @Override + public String resolve(String name) { + try { + String brokerAddr = getBrokerAddr(ProxyContext.createForInner("MQFaultStrategy"), name); + return brokerAddr; + } catch (Exception e) { + return null; + } + } + }, serviceDetector); this.init(); } + // pickup one topic in the topic cache + private Optional pickTopic() { + if (topicCache.asMap().isEmpty()) { + return Optional.absent(); + } + return Optional.of(topicCache.asMap().keySet().iterator().next()); + } + protected void init() { this.appendShutdown(this.scheduledExecutorService::shutdown); this.appendStartAndShutdown(this.mqClientAPIFactory); } + @Override + public void shutdown() throws Exception { + if (this.mqFaultStrategy.isStartDetectorEnable()) { + mqFaultStrategy.shutdown(); + } + } + + @Override + public void start() throws Exception { + if (this.mqFaultStrategy.isStartDetectorEnable()) { + this.mqFaultStrategy.startDetector(); + } + } + + public ClientConfig extractClientConfigFromProxyConfig(ProxyConfig proxyConfig) { + ClientConfig tempClientConfig = new ClientConfig(); + tempClientConfig.setSendLatencyEnable(proxyConfig.getSendLatencyEnable()); + tempClientConfig.setStartDetectorEnable(proxyConfig.getStartDetectorEnable()); + tempClientConfig.setDetectTimeout(proxyConfig.getDetectTimeout()); + tempClientConfig.setDetectInterval(proxyConfig.getDetectInterval()); + return tempClientConfig; + } + + public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, + boolean reachable) { + checkSendFaultToleranceEnable(); + this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); + } + + public void checkSendFaultToleranceEnable() { + boolean hotLatencySwitch = ConfigurationManager.getProxyConfig().isSendLatencyEnable(); + boolean hotDetectorSwitch = ConfigurationManager.getProxyConfig().isStartDetectorEnable(); + this.mqFaultStrategy.setSendLatencyFaultEnable(hotLatencySwitch); + this.mqFaultStrategy.setStartDetectorEnable(hotDetectorSwitch); + } + + public MQFaultStrategy getMqFaultStrategy() { + return this.mqFaultStrategy; + } + public MessageQueueView getAllMessageQueueView(ProxyContext ctx, String topicName) throws Exception { return getCacheMessageQueueWrapper(this.topicCache, topicName); } @@ -132,7 +219,7 @@ protected static boolean isTopicRouteValid(TopicRouteData routeData) { protected MessageQueueView buildMessageQueueView(String topic, TopicRouteData topicRouteData) { if (isTopicRouteValid(topicRouteData)) { - MessageQueueView tmp = new MessageQueueView(topic, topicRouteData); + MessageQueueView tmp = new MessageQueueView(topic, topicRouteData, TopicRouteService.this.getMqFaultStrategy()); log.debug("load topic route from namesrv. topic: {}, queue: {}", topic, tmp); return tmp; } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java index f70c06b8f46..fee3ea87d27 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.proxy.service.sysmessage; import com.alibaba.fastjson.JSON; +import io.netty.channel.Channel; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -73,16 +74,8 @@ protected void init() { ); this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() { @Override - public void handle(ConsumerGroupEvent event, String s, Object... args) { - if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { - if (args == null || args.length < 1) { - return; - } - if (args[0] instanceof ClientChannelInfo) { - ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; - remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText()); - } - } + public void handle(ConsumerGroupEvent event, String group, Object... args) { + processConsumerGroupEvent(event, group, args); } @Override @@ -98,6 +91,18 @@ public void shutdown() throws Exception { super.shutdown(); } + protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) { + if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { + if (args == null || args.length < 1) { + return; + } + if (args[0] instanceof ClientChannelInfo) { + ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; + remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel())); + } + } + } + public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo, ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, Set subList) { @@ -189,7 +194,7 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeCo } RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData()); - RemoteChannel channel = remoteChannelMap.computeIfAbsent(data.getGroup() + "@" + decodedChannel.id().asLongText(), key -> decodedChannel); + RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel); channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute()); ClientChannelInfo clientChannelInfo = new ClientChannelInfo( channel, @@ -228,4 +233,8 @@ private String buildLocalProxyId() { // use local address, remoting port and grpc port to build unique local proxy Id return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort(); } + + private static String buildKey(String group, Channel channel) { + return group + "@" + channel.id().asLongText(); + } } diff --git a/proxy/src/main/resources/rmq.proxy.logback.xml b/proxy/src/main/resources/rmq.proxy.logback.xml index d38827f92d8..f968a45e631 100644 --- a/proxy/src/main/resources/rmq.proxy.logback.xml +++ b/proxy/src/main/resources/rmq.proxy.logback.xml @@ -418,6 +418,11 @@ + + + + + diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java index 49fdfc6a8bf..3c474610518 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java @@ -20,21 +20,32 @@ import apache.rocketmq.v2.AckMessageEntry; import apache.rocketmq.v2.AckMessageRequest; import apache.rocketmq.v2.AckMessageResponse; +import apache.rocketmq.v2.AckMessageResultEntry; import apache.rocketmq.v2.Code; import apache.rocketmq.v2.Resource; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import org.apache.rocketmq.client.consumer.AckResult; import org.apache.rocketmq.client.consumer.AckStatus; import org.apache.rocketmq.proxy.common.ProxyException; import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.apache.rocketmq.proxy.processor.BatchAckResult; +import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; import org.junit.Before; import org.junit.Test; +import org.mockito.stubbing.Answer; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; public class AckMessageActivityTest extends BaseActivityTest { @@ -52,43 +63,197 @@ public void before() throws Throwable { @Test public void testAckMessage() throws Throwable { - when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString())) + ConfigurationManager.getProxyConfig().setEnableBatchAck(false); + + String msg1 = "msg1"; + String msg2 = "msg2"; + String msg3 = "msg3"; + + when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString())) .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired")); AckResult msg2AckResult = new AckResult(); msg2AckResult.setStatus(AckStatus.OK); - when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString())) + when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString())) .thenReturn(CompletableFuture.completedFuture(msg2AckResult)); AckResult msg3AckResult = new AckResult(); msg3AckResult.setStatus(AckStatus.NO_EXIST); - when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString())) + when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString())) .thenReturn(CompletableFuture.completedFuture(msg3AckResult)); - AckMessageResponse response = this.ackMessageActivity.ackMessage( - createContext(), - AckMessageRequest.newBuilder() - .setTopic(Resource.newBuilder().setName(TOPIC).build()) - .setGroup(Resource.newBuilder().setName(GROUP).build()) - .addEntries(AckMessageEntry.newBuilder() - .setMessageId("msg1") - .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) - .build()) - .addEntries(AckMessageEntry.newBuilder() - .setMessageId("msg2") - .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) - .build()) - .addEntries(AckMessageEntry.newBuilder() - .setMessageId("msg3") - .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) - .build()) - .build() - ).get(); - - assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); - assertEquals(3, response.getEntriesCount()); - assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); - assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); - assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); + { + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(msg1) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) + .build()) + .build() + ).get(); + assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); + } + { + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(msg2) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) + .build()) + .build() + ).get(); + assertEquals(Code.OK, response.getStatus().getCode()); + } + { + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(msg3) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) + .build()) + .build() + ).get(); + assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); + } + { + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(msg1) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) + .build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(msg2) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(msg3) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .build() + ).get(); + + assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); + assertEquals(3, response.getEntriesCount()); + assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); + assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); + assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); + } + } + + @Test + public void testAckMessageInBatch() throws Throwable { + ConfigurationManager.getProxyConfig().setEnableBatchAck(true); + + String successMessageId = "msg1"; + String notOkMessageId = "msg2"; + String exceptionMessageId = "msg3"; + + doAnswer((Answer>>) invocation -> { + List receiptHandleMessageList = invocation.getArgument(1, List.class); + List batchAckResultList = new ArrayList<>(); + for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) { + BatchAckResult batchAckResult; + if (receiptHandleMessage.getMessageId().equals(successMessageId)) { + AckResult ackResult = new AckResult(); + ackResult.setStatus(AckStatus.OK); + batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); + } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) { + AckResult ackResult = new AckResult(); + ackResult.setStatus(AckStatus.NO_EXIST); + batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); + } else { + batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "")); + } + batchAckResultList.add(batchAckResult); + } + return CompletableFuture.completedFuture(batchAckResultList); + }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString()); + + { + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(successMessageId) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .build() + ).get(); + assertEquals(Code.OK, response.getStatus().getCode()); + } + { + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(notOkMessageId) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .build() + ).get(); + assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); + } + { + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(exceptionMessageId) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .build() + ).get(); + assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); + } + { + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(successMessageId) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(notOkMessageId) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId(exceptionMessageId) + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .build() + ).get(); + + assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); + assertEquals(3, response.getEntriesCount()); + Map msgCode = new HashMap<>(); + for (AckMessageResultEntry entry : response.getEntriesList()) { + msgCode.put(entry.getMessageId(), entry.getStatus().getCode()); + } + assertEquals(Code.OK, msgCode.get(successMessageId)); + assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId)); + assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId)); + } } } \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java index 7fd9a9ffdf0..77ae5e4d111 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java @@ -93,7 +93,6 @@ public void testReceiveMessagePollingTime() { pollTimeCaptor.capture(), anyInt(), any(), anyBoolean(), any(), isNull(), anyLong())) .thenReturn(CompletableFuture.completedFuture(new PopResult(PopStatus.NO_NEW_MSG, Collections.emptyList()))); - ProxyContext context = createContext(); context.setRemainingMs(1L); this.receiveMessageActivity.receiveMessage( @@ -274,7 +273,7 @@ private Code getResponseCodeFromReceiveMessageResponseList(List queueDatas = new ArrayList<>(); for (int i = 0; i < 2; i++) { @@ -298,7 +297,7 @@ public void testReceiveMessageQueueSelector() { } topicRouteData.setBrokerDatas(brokerDatas); - MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector(""); AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java index 588423bb93f..4882a5ed8b7 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java @@ -35,6 +35,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.latency.MQFaultStrategy; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; import org.apache.rocketmq.common.MixAll; @@ -49,6 +51,7 @@ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; import org.apache.rocketmq.remoting.protocol.route.BrokerData; import org.apache.rocketmq.remoting.protocol.route.QueueData; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; @@ -62,15 +65,19 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class SendMessageActivityTest extends BaseActivityTest { protected static final String BROKER_NAME = "broker"; + protected static final String BROKER_NAME2 = "broker2"; protected static final String CLUSTER_NAME = "cluster"; protected static final String BROKER_ADDR = "127.0.0.1:10911"; + protected static final String BROKER_ADDR2 = "127.0.0.1:10912"; private static final String TOPIC = "topic"; private static final String CONSUMER_GROUP = "consumerGroup"; + MQFaultStrategy mqFaultStrategy; private SendMessageActivity sendMessageActivity; @@ -262,7 +269,7 @@ public void testTxMessage() { } @Test - public void testSendOrderMessageQueueSelector() { + public void testSendOrderMessageQueueSelector() throws Exception { TopicRouteData topicRouteData = new TopicRouteData(); QueueData queueData = new QueueData(); BrokerData brokerData = new BrokerData(); @@ -277,7 +284,7 @@ public void testSendOrderMessageQueueSelector() { brokerData.setBrokerAddrs(brokerAddrs); topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); - MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); SendMessageActivity.SendMessageQueueSelector selector1 = new SendMessageActivity.SendMessageQueueSelector( SendMessageRequest.newBuilder() .addMessages(Message.newBuilder() @@ -288,6 +295,12 @@ public void testSendOrderMessageQueueSelector() { .build() ); + TopicRouteService topicRouteService = mock(TopicRouteService.class); + MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class); + when(topicRouteService.getAllMessageQueueView(any(), any())).thenReturn(messageQueueView); + when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); + when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false); + SendMessageActivity.SendMessageQueueSelector selector2 = new SendMessageActivity.SendMessageQueueSelector( SendMessageRequest.newBuilder() .addMessages(Message.newBuilder() @@ -328,12 +341,17 @@ public void testSendNormalMessageQueueSelector() { brokerData.setBrokerAddrs(brokerAddrs); topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); - MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); + SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( SendMessageRequest.newBuilder() .addMessages(Message.newBuilder().build()) .build() ); + TopicRouteService topicRouteService = mock(TopicRouteService.class); + MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class); + when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); + when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false); + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy()); AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); @@ -343,6 +361,45 @@ public void testSendNormalMessageQueueSelector() { assertNotEquals(firstSelect, secondSelect); } + @Test + public void testSendNormalMessageQueueSelectorPipeLine() throws Exception { + TopicRouteData topicRouteData = new TopicRouteData(); + int queueNums = 2; + + QueueData queueData = createQueueData(BROKER_NAME, queueNums); + QueueData queueData2 = createQueueData(BROKER_NAME2, queueNums); + topicRouteData.setQueueDatas(Lists.newArrayList(queueData,queueData2)); + + + BrokerData brokerData = createBrokerData(CLUSTER_NAME, BROKER_NAME, BROKER_ADDR); + BrokerData brokerData2 = createBrokerData(CLUSTER_NAME, BROKER_NAME2, BROKER_ADDR2); + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData, brokerData2)); + + SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder().build()) + .build() + ); + + ClientConfig cc = new ClientConfig(); + this.mqFaultStrategy = new MQFaultStrategy(cc, null, null); + mqFaultStrategy.setSendLatencyFaultEnable(true); + mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, true); + mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, false); + + TopicRouteService topicRouteService = mock(TopicRouteService.class); + when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy()); + + + AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); + assertEquals(firstSelect.getBrokerName(), BROKER_NAME2); + + mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, false); + mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, true); + AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); + assertEquals(secondSelect.getBrokerName(), BROKER_NAME); + } @Test public void testParameterValidate() { // too large message body @@ -850,4 +907,23 @@ private static String createStr(int len) { } return sb.toString(); } + + private static QueueData createQueueData(String brokerName, int writeQueueNums) { + QueueData queueData = new QueueData(); + queueData.setBrokerName(brokerName); + queueData.setWriteQueueNums(writeQueueNums); + queueData.setPerm(PermName.PERM_WRITE); + return queueData; + } + + private static BrokerData createBrokerData(String clusterName, String brokerName, String brokerAddrs) { + BrokerData brokerData = new BrokerData(); + brokerData.setCluster(clusterName); + brokerData.setBrokerName(brokerName); + HashMap brokerAddrsMap = new HashMap<>(); + brokerAddrsMap.put(MixAll.MASTER_ID, brokerAddrs); + brokerData.setBrokerAddrs(brokerAddrsMap); + + return brokerData; + } } diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java index 5c1ea9627e6..072630e3947 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java @@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest { protected ProxyRelayService proxyRelayService; @Mock protected MetadataService metadataService; - @Mock - protected ProducerProcessor producerProcessor; - @Mock - protected ConsumerProcessor consumerProcessor; - @Mock - protected TransactionProcessor transactionProcessor; - @Mock - protected ClientProcessor clientProcessor; public void before() throws Throwable { super.before(); @@ -92,6 +84,13 @@ protected static ProxyContext createContext() { } protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) { + return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(), + RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), + RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker"); + } + + protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime, + long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) { MessageExt messageExt = new MessageExt(); messageExt.setTopic(topic); messageExt.setTags(tags); @@ -100,8 +99,7 @@ protected static MessageExt createMessageExt(String topic, String tags, int reco messageExt.setMsgId(MessageClientIDSetter.createUniqID()); messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, - ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime, - RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); + ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset)); return messageExt; } diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java index 717e86fc056..db268a06e6a 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java @@ -20,8 +20,11 @@ import com.google.common.collect.Sets; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; @@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.common.utils.FutureUtils; import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; import org.apache.rocketmq.proxy.service.route.MessageQueueView; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -50,16 +56,22 @@ import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; +import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class ConsumerProcessorTest extends BaseProcessorTest { @@ -162,6 +174,109 @@ public void testAckMessage() throws Throwable { assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); } + @Test + public void testBatchAckExpireMessage() throws Throwable { + String brokerName1 = "brokerName1"; + + List receiptHandleMessageList = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, + 0, 0, 0, i, brokerName1); + ReceiptHandle expireHandle = create(expireMessage); + receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); + } + + List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); + + verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); + assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); + for (BatchAckResult batchAckResult : batchAckResultList) { + assertNull(batchAckResult.getAckResult()); + assertNotNull(batchAckResult.getProxyException()); + assertNotNull(batchAckResult.getReceiptHandleMessage()); + } + + } + + @Test + public void testBatchAckMessage() throws Throwable { + String brokerName1 = "brokerName1"; + String brokerName2 = "brokerName2"; + String errThrowBrokerName = "errThrowBrokerName"; + MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, + 0, 0, 0, 0, brokerName1); + ReceiptHandle expireHandle = create(expireMessage); + + List receiptHandleMessageList = new ArrayList<>(); + receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); + List broker1Msg = new ArrayList<>(); + List broker2Msg = new ArrayList<>(); + + long now = System.currentTimeMillis(); + int msgNum = 3; + for (int i = 0; i < msgNum; i++) { + MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, + 0, 0, 0, i + 1, brokerName1); + ReceiptHandle brokerHandle = create(brokerMessage); + receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); + broker1Msg.add(brokerMessage.getMsgId()); + } + for (int i = 0; i < msgNum; i++) { + MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, + 0, 0, 0, i + 1, brokerName2); + ReceiptHandle brokerHandle = create(brokerMessage); + receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); + broker2Msg.add(brokerMessage.getMsgId()); + } + + // for this message, will throw exception in batchAckMessage + MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now, + 0, 0, 0, 0, errThrowBrokerName); + ReceiptHandle errThrowHandle = create(errThrowMessage); + receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId())); + + Collections.shuffle(receiptHandleMessageList); + + doAnswer((Answer>) invocation -> { + List handleMessageList = invocation.getArgument(1, List.class); + AckResult ackResult = new AckResult(); + String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName(); + if (brokerName.equals(brokerName1)) { + ackResult.setStatus(AckStatus.OK); + } else if (brokerName.equals(brokerName2)) { + ackResult.setStatus(AckStatus.NO_EXIST); + } else { + return FutureUtils.completeExceptionally(new RuntimeException()); + } + + return CompletableFuture.completedFuture(ackResult); + }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); + + List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); + assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); + + // check ackResult for each msg + Map msgBatchAckResult = new HashMap<>(); + for (BatchAckResult batchAckResult : batchAckResultList) { + msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult); + } + for (String msgId : broker1Msg) { + assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus()); + assertNull(msgBatchAckResult.get(msgId).getProxyException()); + } + for (String msgId : broker2Msg) { + assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus()); + assertNull(msgBatchAckResult.get(msgId).getProxyException()); + } + assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException()); + assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode()); + assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult()); + + assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException()); + assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode()); + assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult()); + } + @Test public void testChangeInvisibleTime() throws Throwable { ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java index c97bd5a72b4..ca6fe909e28 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java @@ -78,7 +78,7 @@ public void before() throws Throwable { topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); when(this.topicRouteService.getAllMessageQueueView(any(), eq(ERR_TOPIC))).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "")); - when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); - when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData)); + when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null)); + when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData, null)); } } diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java index 77a119a296c..e2d05b0f5a8 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -85,6 +86,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; @RunWith(MockitoJUnitRunner.class) public class MQClientAPIExtTest { @@ -109,13 +111,9 @@ public void init() throws Exception { @Test public void testSendHeartbeatAsync() throws Exception { - doAnswer((Answer) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture future = new CompletableFuture<>(); + future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get()); } @@ -123,20 +121,16 @@ public void testSendHeartbeatAsync() throws Exception { @Test public void testSendMessageAsync() throws Exception { AtomicReference msgIdRef = new AtomicReference<>(); - doAnswer((Answer) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); - sendMessageResponseHeader.setMsgId(msgIdRef.get()); - sendMessageResponseHeader.setQueueId(0); - sendMessageResponseHeader.setQueueOffset(1L); - response.setCode(ResponseCode.SUCCESS); - response.makeCustomHeaderToNet(); - responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture future = new CompletableFuture<>(); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + sendMessageResponseHeader.setMsgId(msgIdRef.get()); + sendMessageResponseHeader.setQueueId(0); + sendMessageResponseHeader.setQueueOffset(1L); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + future.complete(response); + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); MessageExt messageExt = createMessage(); msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt)); @@ -150,20 +144,16 @@ public void testSendMessageAsync() throws Exception { @Test public void testSendMessageListAsync() throws Exception { - doAnswer((Answer) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); - sendMessageResponseHeader.setMsgId(""); - sendMessageResponseHeader.setQueueId(0); - sendMessageResponseHeader.setQueueOffset(1L); - response.setCode(ResponseCode.SUCCESS); - response.makeCustomHeaderToNet(); - responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture future = new CompletableFuture<>(); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + sendMessageResponseHeader.setMsgId(""); + sendMessageResponseHeader.setQueueId(0); + sendMessageResponseHeader.setQueueOffset(1L); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + future.complete(response); + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); List messageExtList = new ArrayList<>(); StringBuilder sb = new StringBuilder(); @@ -182,13 +172,9 @@ public void testSendMessageListAsync() throws Exception { @Test public void testSendMessageBackAsync() throws Exception { - doAnswer((Answer) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture future = new CompletableFuture<>(); + future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT) .get(); @@ -220,6 +206,18 @@ public void testAckMessageAsync() throws Exception { assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get()); } + @Test + public void testBatchAckMessageAsync() throws Exception { + AckResult ackResult = new AckResult(); + doAnswer((Answer) mock -> { + AckCallback ackCallback = mock.getArgument(2); + ackCallback.onSuccess(ackResult); + return null; + }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any()); + + assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get()); + } + @Test public void testChangeInvisibleTimeAsync() throws Exception { AckResult ackResult = new AckResult(); @@ -273,7 +271,7 @@ public void testGetConsumerListByGroupAsync() throws Exception { body.setConsumerIdList(clientIds); response.setBody(body.encode()); responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); + invokeCallback.operationSucceed(responseFuture.getResponseCommand()); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); @@ -290,7 +288,7 @@ public void testGetEmptyConsumerListByGroupAsync() throws Exception { response.setCode(ResponseCode.SYSTEM_ERROR); response.makeCustomHeaderToNet(); responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); + invokeCallback.operationSucceed(responseFuture.getResponseCommand()); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); @@ -310,7 +308,7 @@ public void testGetMaxOffsetAsync() throws Exception { response.setCode(ResponseCode.SUCCESS); response.makeCustomHeaderToNet(); responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); + invokeCallback.operationSucceed(responseFuture.getResponseCommand()); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); @@ -323,18 +321,15 @@ public void testGetMaxOffsetAsync() throws Exception { @Test public void testSearchOffsetAsync() throws Exception { long offset = ThreadLocalRandom.current().nextLong(); - doAnswer((Answer) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); - SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); - responseHeader.setOffset(offset); - response.setCode(ResponseCode.SUCCESS); - response.makeCustomHeaderToNet(); - responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture future = new CompletableFuture<>(); + RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); + SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + future.complete(response); + + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader(); requestHeader.setTopic(TOPIC); diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java index e44ed28f4a6..d150f87c409 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java @@ -30,12 +30,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest { public void testReadMessageQueue() { queueData.setPerm(PermName.PERM_READ); queueData.setReadQueueNums(0); - MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); + MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true); assertTrue(messageQueueSelector.getQueues().isEmpty()); queueData.setPerm(PermName.PERM_READ); queueData.setReadQueueNums(3); - messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); + messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true); assertEquals(3, messageQueueSelector.getQueues().size()); assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { @@ -58,12 +58,12 @@ public void testReadMessageQueue() { public void testWriteMessageQueue() { queueData.setPerm(PermName.PERM_WRITE); queueData.setReadQueueNums(0); - MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); + MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false); assertTrue(messageQueueSelector.getQueues().isEmpty()); queueData.setPerm(PermName.PERM_WRITE); queueData.setWriteQueueNums(3); - messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); + messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false); assertEquals(3, messageQueueSelector.getQueues().size()); assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java index c67f4953d89..9a2c5e3437d 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java @@ -27,6 +27,7 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelId; import java.time.Duration; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -35,6 +36,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupEvent; import org.apache.rocketmq.broker.client.ConsumerManager; import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; @@ -132,7 +134,7 @@ public void before() throws Throwable { brokerAddr.put(0L, "127.0.0.1:10911"); brokerData.setBrokerAddrs(brokerAddr); topicRouteData.getBrokerDatas().add(brokerData); - MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData); + MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData, null); when(this.topicRouteService.getAllMessageQueueView(any(), anyString())).thenReturn(messageQueueView); } } @@ -320,6 +322,72 @@ public void testSyncRemotingChannel() throws Exception { } } + @Test + public void testProcessConsumerGroupEventForRemoting() { + String consumerGroup = "consumerGroup"; + Channel channel = createMockChannel(); + RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class); + RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet()); + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + remotingChannel, + clientId, + LanguageCode.JAVA, + 4 + ); + + testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); + } + + @Test + public void testProcessConsumerGroupEventForGrpcV2() { + String consumerGroup = "consumerGroup"; + GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class); + GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class); + GrpcClientChannel grpcClientChannel = new GrpcClientChannel( + proxyRelayService, grpcClientSettingsManager, grpcChannelManager, + ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress), + clientId); + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + grpcClientChannel, + clientId, + LanguageCode.JAVA, + 5 + ); + + testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); + } + + private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) { + HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null); + SendResult okSendResult = new SendResult(); + okSendResult.setSendStatus(SendStatus.SEND_OK); + + ArgumentCaptor messageArgumentCaptor = ArgumentCaptor.forClass(Message.class); + doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt) + .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong()); + + heartbeatSyncer.onConsumerRegister( + consumerGroup, + clientChannelInfo, + ConsumeType.CONSUME_PASSIVELY, + MessageModel.CLUSTERING, + ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, + Collections.emptySet() + ); + await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1); + + // change local serve addr, to simulate other proxy receive messages + heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10); + ArgumentCaptor channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); + doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean()); + + heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null); + assertEquals(1, heartbeatSyncer.remoteChannelMap.size()); + + heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue()); + assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty()); + } + private MessageExt convertFromMessage(Message message) { MessageExt messageExt = new MessageExt(); messageExt.setTopic(message.getTopic()); diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java index a0063544ecf..91af74cbefc 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java @@ -64,7 +64,7 @@ public void before() throws Throwable { this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager, this.mqClientAPIFactory); - MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); when(this.topicRouteService.getAllMessageQueueView(any(), anyString())) .thenReturn(messageQueueView); @@ -127,7 +127,7 @@ public void testScanProducerHeartBeat() throws Exception { brokerData.setBrokerAddrs(brokerAddrs); topicRouteData.getQueueDatas().add(queueData); topicRouteData.getBrokerDatas().add(brokerData); - when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); + when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null)); TopicRouteData clusterTopicRouteData = new TopicRouteData(); QueueData clusterQueueData = new QueueData(); @@ -141,7 +141,7 @@ public void testScanProducerHeartBeat() throws Exception { brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR); clusterBrokerData.setBrokerAddrs(brokerAddrs); clusterTopicRouteData.setBrokerDatas(Lists.newArrayList(clusterBrokerData)); - when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData)); + when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData, null)); TopicRouteData clusterTopicRouteData2 = new TopicRouteData(); QueueData clusterQueueData2 = new QueueData(); @@ -155,7 +155,7 @@ public void testScanProducerHeartBeat() throws Exception { brokerAddrs.put(MixAll.MASTER_ID, brokerAddr2); clusterBrokerData2.setBrokerAddrs(brokerAddrs); clusterTopicRouteData2.setBrokerDatas(Lists.newArrayList(clusterBrokerData2)); - when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2)); + when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2, null)); ConfigurationManager.getProxyConfig().setTransactionHeartbeatBatchNum(2); this.clusterTransactionService.start(); diff --git a/remoting/pom.xml b/remoting/pom.xml index 8a43c5c30c0..f7848068055 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java index ce78fa923ff..6be49174571 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java @@ -17,7 +17,22 @@ package org.apache.rocketmq.remoting; import org.apache.rocketmq.remoting.netty.ResponseFuture; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; public interface InvokeCallback { + /** + * This method is expected to be invoked after {@link #operationSucceed(RemotingCommand)} + * or {@link #operationFail(Throwable)} + * + * @param responseFuture the returned object contains response or exception + */ void operationComplete(final ResponseFuture responseFuture); + + default void operationSucceed(final RemotingCommand response) { + + } + + default void operationFail(final Throwable throwable) { + + } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java index ff0b3df95a6..c8389eedb18 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java @@ -20,11 +20,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.RemotingCommand; public interface RemotingClient extends RemotingService { @@ -51,18 +51,21 @@ default CompletableFuture invoke(final String addr, final Remot final long timeoutMillis) { CompletableFuture future = new CompletableFuture<>(); try { - invokeAsync(addr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { + invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { future.complete(response); - } else { - if (!responseFuture.isSendRequestOK()) { - future.completeExceptionally(new RemotingSendRequestException(addr, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - future.completeExceptionally(new RemotingTimeoutException(addr, timeoutMillis, responseFuture.getCause())); - } else { - future.completeExceptionally(new RemotingException(request.toString(), responseFuture.getCause())); - } + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } }); } catch (Throwable t) { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java index b2e7df75491..c28288786a3 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java @@ -53,6 +53,12 @@ public class NettyClientConfig { private boolean disableCallbackExecutor = false; private boolean disableNettyWorkerGroup = false; + private long maxReconnectIntervalTimeSeconds = 60; + + private boolean enableReconnectForGoAway = true; + + private boolean enableTransparentRetry = true; + public boolean isClientCloseSocketIfTimeout() { return clientCloseSocketIfTimeout; } @@ -181,6 +187,30 @@ public void setDisableNettyWorkerGroup(boolean disableNettyWorkerGroup) { this.disableNettyWorkerGroup = disableNettyWorkerGroup; } + public long getMaxReconnectIntervalTimeSeconds() { + return maxReconnectIntervalTimeSeconds; + } + + public void setMaxReconnectIntervalTimeSeconds(long maxReconnectIntervalTimeSeconds) { + this.maxReconnectIntervalTimeSeconds = maxReconnectIntervalTimeSeconds; + } + + public boolean isEnableReconnectForGoAway() { + return enableReconnectForGoAway; + } + + public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) { + this.enableReconnectForGoAway = enableReconnectForGoAway; + } + + public boolean isEnableTransparentRetry() { + return enableTransparentRetry; + } + + public void setEnableTransparentRetry(boolean enableTransparentRetry) { + this.enableTransparentRetry = enableTransparentRetry; + } + public String getSocksProxyConfig() { return socksProxyConfig; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 44d6a3df4c4..07ace28ea54 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -23,23 +23,28 @@ import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.Future; import io.opentelemetry.api.common.AttributesBuilder; -import java.net.SocketAddress; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.rocketmq.common.AbortProcessException; +import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.UtilAll; @@ -57,6 +62,7 @@ import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_IS_LONG_POLLING; import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE; @@ -117,6 +123,8 @@ public abstract class NettyRemotingAbstract { */ protected List rpcHooks = new ArrayList<>(); + protected AtomicBoolean isShuttingDown = new AtomicBoolean(false); + static { NettyLogger.initNettyLogger(); } @@ -125,7 +133,7 @@ public abstract class NettyRemotingAbstract { * Constructor, specifying capacity of one-way and asynchronous semaphores. * * @param permitsOneway Number of permits for one-way requests. - * @param permitsAsync Number of permits for asynchronous requests. + * @param permitsAsync Number of permits for asynchronous requests. */ public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { this.semaphoreOneway = new Semaphore(permitsOneway, true); @@ -261,6 +269,16 @@ public void processRequestCommand(final ChannelHandlerContext ctx, final Remotin Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque); + if (isShuttingDown.get()) { + if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) { + final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY, + "please go away"); + response.setOpaque(opaque); + writeResponse(ctx.channel(), cmd, response); + return; + } + } + if (pair.getObject1().rejectRequest()) { final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY, "[REJECTREQUEST]system busy, start flow control for a while"); @@ -367,8 +385,7 @@ public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cm responseFuture.release(); } } else { - log.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - log.warn(cmd.toString()); + log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); } } @@ -467,57 +484,68 @@ public void scanResponseTable() { public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { - //get the request id - final int opaque = request.getOpaque(); - try { - final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null); - this.responseTable.put(opaque, responseFuture); - final SocketAddress addr = channel.remoteAddress(); - channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { - if (f.isSuccess()) { - responseFuture.setSendRequestOK(true); - return; - } - - responseFuture.setSendRequestOK(false); - responseTable.remove(opaque); - responseFuture.setCause(f.cause()); - responseFuture.putResponse(null); - log.warn("Failed to write a request command to {}, caused by underlying I/O operation failure", addr); - }); + return invokeImpl(channel, request, timeoutMillis).thenApply(ResponseFuture::getResponseCommand) + .get(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw new RemotingSendRequestException(channel.remoteAddress().toString(), e.getCause()); + } catch (TimeoutException e) { + throw new RemotingTimeoutException(channel.remoteAddress().toString(), timeoutMillis, e.getCause()); + } + } - RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); - if (null == responseCommand) { - if (responseFuture.isSendRequestOK()) { - throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis, - responseFuture.getCause()); - } else { - throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause()); - } + public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand request, + final long timeoutMillis) { + String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); + doBeforeRpcHooks(channelRemoteAddr, request); + return invoke0(channel, request, timeoutMillis).whenComplete((v, t) -> { + if (t == null) { + doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand()); } - - return responseCommand; - } finally { - this.responseTable.remove(opaque); - } + }); } - public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, - final InvokeCallback invokeCallback) - throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + protected CompletableFuture invoke0(final Channel channel, final RemotingCommand request, + final long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); long beginStartTime = System.currentTimeMillis(); final int opaque = request.getOpaque(); - boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + + boolean acquired; + try { + acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (Throwable t) { + future.completeExceptionally(t); + return future; + } if (acquired) { final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); long costTime = System.currentTimeMillis() - beginStartTime; if (timeoutMillis < costTime) { once.release(); - throw new RemotingTimeoutException("invokeAsyncImpl call timeout"); + future.completeExceptionally(new RemotingTimeoutException("invokeAsyncImpl call timeout")); + return future; } - final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once); + AtomicReference responseFutureReference = new AtomicReference<>(); + final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, request, timeoutMillis - costTime, + new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + future.complete(responseFutureReference.get()); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); + } + }, once); + responseFutureReference.set(responseFuture); this.responseTable.put(opaque, responseFuture); try { channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { @@ -528,14 +556,17 @@ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request requestFail(opaque); log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); }); + return future; } catch (Exception e) { + responseTable.remove(opaque); responseFuture.release(); log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); - throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); + future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e)); + return future; } } else { if (timeoutMillis <= 0) { - throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"); + future.completeExceptionally(new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast")); } else { String info = String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", @@ -544,11 +575,31 @@ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request this.semaphoreAsync.availablePermits() ); log.warn(info); - throw new RemotingTimeoutException(info); + future.completeExceptionally(new RemotingTimeoutException(info)); } + return future; } } + public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, + final InvokeCallback invokeCallback) { + invokeImpl(channel, request, timeoutMillis) + .whenComplete((v, t) -> { + if (t == null) { + invokeCallback.operationComplete(v); + } else { + ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, timeoutMillis, null, null); + responseFuture.setCause(t); + invokeCallback.operationComplete(responseFuture); + } + }) + .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand())) + .exceptionally(t -> { + invokeCallback.operationFail(t); + return null; + }); + } + private void requestFail(final int opaque) { ResponseFuture responseFuture = responseTable.remove(opaque); if (responseFuture != null) { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 8491f4354c6..340daee67e3 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -18,6 +18,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; +import com.google.common.base.Stopwatch; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.Channel; @@ -48,6 +49,7 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.security.cert.CertificateException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -57,20 +59,22 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; @@ -82,6 +86,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.proxy.SocksProxyConfig; public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { @@ -97,6 +102,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti private final Map proxyMap = new HashMap<>(); private final ConcurrentHashMap bootstrapMap = new ConcurrentHashMap<>(); private final ConcurrentMap channelTables = new ConcurrentHashMap<>(); + private final ConcurrentMap channelWrapperTables = new ConcurrentHashMap<>(); private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, "ClientHouseKeepingService")); @@ -142,7 +148,7 @@ public NettyRemotingClient(final NettyClientConfig nettyClientConfig, this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactoryImpl("NettyClientPublicExecutor_")); - this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, + this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("NettyClientScan_thread_")); if (eventLoopGroup != null) { @@ -229,6 +235,8 @@ public void initChannel(SocketChannel ch) throws Exception { handler.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); } + nettyEventExecutor.start(); + TimerTask timerTaskScanResponseTable = new TimerTask() { @Override public void run(Timeout timeout) { @@ -354,9 +362,10 @@ public void shutdown() { this.timer.stop(); for (String addr : this.channelTables.keySet()) { - this.closeChannel(addr, this.channelTables.get(addr).getChannel()); + this.channelTables.get(addr).close(); } + this.channelWrapperTables.clear(); this.channelTables.clear(); this.eventLoopGroupWorker.shutdownGracefully(); @@ -414,7 +423,10 @@ public void closeChannel(final String addr, final Channel channel) { } if (removeItemFromTable) { - this.channelTables.remove(addrRemote); + ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); + if (channelWrapper != null && channelWrapper.tryClose(channel)) { + this.channelTables.remove(addrRemote); + } LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); } @@ -461,7 +473,10 @@ public void closeChannel(final Channel channel) { } if (removeItemFromTable) { - this.channelTables.remove(addrRemote); + ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); + if (channelWrapper != null && channelWrapper.tryClose(channel)) { + this.channelTables.remove(addrRemote); + } LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); RemotingHelper.closeChannel(channel); } @@ -509,7 +524,7 @@ public void updateNameServerAddressList(List addrs) { if (addr.contains(namesrvAddr)) { ChannelWrapper channelWrapper = this.channelTables.get(addr); if (channelWrapper != null) { - closeChannel(channelWrapper.getChannel()); + channelWrapper.close(); } } } @@ -527,15 +542,13 @@ public RemotingCommand invokeSync(String addr, final RemotingCommand request, lo if (channel != null && channel.isActive()) { long left = timeoutMillis; try { - doBeforeRpcHooks(channelRemoteAddr, request); long costTime = System.currentTimeMillis() - beginStartTime; left -= costTime; if (left <= 0) { throw new RemotingTimeoutException("invokeSync call the addr[" + channelRemoteAddr + "] timeout"); } RemotingCommand response = this.invokeSyncImpl(channel, request, left); - doAfterRpcHooks(channelRemoteAddr, request, response); - this.updateChannelLastResponseTime(addr); + updateChannelLastResponseTime(addr); return response; } catch (RemotingSendRequestException e) { LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr); @@ -689,8 +702,9 @@ private Channel createChannel(final String addr) throws InterruptedException { ChannelFuture channelFuture = fetchBootstrap(addr) .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr); - cw = new ChannelWrapper(channelFuture); + cw = new ChannelWrapper(addr, channelFuture); this.channelTables.put(addr, cw); + this.channelWrapperTables.put(channelFuture.channel(), cw); } } catch (Exception e) { LOGGER.error("createChannel: create channel exception", e); @@ -702,20 +716,25 @@ private Channel createChannel(final String addr) throws InterruptedException { } if (cw != null) { - ChannelFuture channelFuture = cw.getChannelFuture(); - if (channelFuture.awaitUninterruptibly(this.nettyClientConfig.getConnectTimeoutMillis())) { - if (cw.isOK()) { - LOGGER.info("createChannel: connect remote host[{}] success, {}", addr, channelFuture.toString()); - return cw.getChannel(); - } else { - LOGGER.warn("createChannel: connect remote host[" + addr + "] failed, " + channelFuture.toString()); - } + return waitChannelFuture(addr, cw); + } + + return null; + } + + private Channel waitChannelFuture(String addr, ChannelWrapper cw) { + ChannelFuture channelFuture = cw.getChannelFuture(); + if (channelFuture.awaitUninterruptibly(this.nettyClientConfig.getConnectTimeoutMillis())) { + if (cw.isOK()) { + LOGGER.info("createChannel: connect remote host[{}] success, {}", addr, channelFuture.toString()); + return cw.getChannel(); } else { - LOGGER.warn("createChannel: connect remote host[{}] timeout {}ms, {}", addr, this.nettyClientConfig.getConnectTimeoutMillis(), - channelFuture.toString()); + LOGGER.warn("createChannel: connect remote host[{}] failed, {}", addr, channelFuture.toString()); } + } else { + LOGGER.warn("createChannel: connect remote host[{}] timeout {}ms, {}", addr, this.nettyClientConfig.getConnectTimeoutMillis(), + channelFuture.toString()); } - return null; } @@ -727,18 +746,11 @@ public void invokeAsync(String addr, RemotingCommand request, long timeoutMillis final Channel channel = this.getAndCreateChannel(addr); String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); if (channel != null && channel.isActive()) { - try { - doBeforeRpcHooks(channelRemoteAddr, request); - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeoutMillis < costTime) { - throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); - } - this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); - } catch (RemotingSendRequestException e) { - LOGGER.warn("invokeAsync: send request exception, so close the channel[{}]", channelRemoteAddr); - this.closeChannel(addr, channel); - throw e; + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeoutMillis < costTime) { + throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); } + this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); } else { this.closeChannel(addr, channel); throw new RemotingConnectException(addr); @@ -765,6 +777,70 @@ public void invokeOneway(String addr, RemotingCommand request, long timeoutMilli } } + @Override + public CompletableFuture invoke(String addr, RemotingCommand request, + long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + final Channel channel = this.getAndCreateChannel(addr); + if (channel != null && channel.isActive()) { + return invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> { + if (t == null) { + updateChannelLastResponseTime(addr); + } + }).thenApply(ResponseFuture::getResponseCommand); + } else { + this.closeChannel(addr, channel); + future.completeExceptionally(new RemotingConnectException(addr)); + } + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + @Override + public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand request, + final long timeoutMillis) { + Stopwatch stopwatch = Stopwatch.createStarted(); + return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response.getCode() == ResponseCode.GO_AWAY) { + if (nettyClientConfig.isEnableReconnectForGoAway()) { + ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> { + try { + if (channelWrapper0.reconnect()) { + LOGGER.info("Receive go away from channel {}, recreate the channel", channel0); + channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0); + } + } catch (Throwable t) { + LOGGER.error("Channel {} reconnect error", channelWrapper0, t); + } + return channelWrapper0; + }); + if (channelWrapper != null) { + if (nettyClientConfig.isEnableTransparentRetry()) { + long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); + stopwatch.stop(); + RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); + retryRequest.setBody(request.getBody()); + Channel retryChannel; + if (channelWrapper.isOK()) { + retryChannel = channelWrapper.getChannel(); + } else { + retryChannel = waitChannelFuture(channelWrapper.getChannelAddress(), channelWrapper); + } + if (retryChannel != null && channel != retryChannel) { + return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration); + } + } + } + } + } + return CompletableFuture.completedFuture(responseFuture); + }); + } + @Override public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { ExecutorService executorThis = executor; @@ -884,30 +960,41 @@ public void run() { } } - static class ChannelWrapper { - private final ChannelFuture channelFuture; + class ChannelWrapper { + private final ReentrantReadWriteLock lock; + private ChannelFuture channelFuture; // only affected by sync or async request, oneway is not included. + private ChannelFuture channelToClose; private long lastResponseTime; + private volatile long lastReconnectTimestamp = 0L; + private final String channelAddress; - public ChannelWrapper(ChannelFuture channelFuture) { + public ChannelWrapper(String address, ChannelFuture channelFuture) { + this.lock = new ReentrantReadWriteLock(); this.channelFuture = channelFuture; this.lastResponseTime = System.currentTimeMillis(); + this.channelAddress = address; } public boolean isOK() { - return this.channelFuture.channel() != null && this.channelFuture.channel().isActive(); + return getChannel() != null && getChannel().isActive(); } public boolean isWritable() { - return this.channelFuture.channel().isWritable(); + return getChannel().isWritable(); } private Channel getChannel() { - return this.channelFuture.channel(); + return getChannelFuture().channel(); } public ChannelFuture getChannelFuture() { - return channelFuture; + lock.readLock().lock(); + try { + return this.channelFuture; + } finally { + lock.readLock().unlock(); + } } public long getLastResponseTime() { @@ -917,6 +1004,56 @@ public long getLastResponseTime() { public void updateLastResponseTime() { this.lastResponseTime = System.currentTimeMillis(); } + + public String getChannelAddress() { + return channelAddress; + } + + public boolean reconnect() { + if (lock.writeLock().tryLock()) { + try { + if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) { + channelToClose = channelFuture; + String[] hostAndPort = getHostAndPort(channelAddress); + channelFuture = fetchBootstrap(channelAddress) + .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); + lastReconnectTimestamp = System.currentTimeMillis(); + return true; + } + } finally { + lock.writeLock().unlock(); + } + } + return false; + } + + public boolean tryClose(Channel channel) { + try { + lock.readLock().lock(); + if (channelFuture != null) { + if (channelFuture.channel().equals(channel)) { + return true; + } + } + } finally { + lock.readLock().unlock(); + } + return false; + } + + public void close() { + try { + lock.writeLock().lock(); + if (channelFuture != null) { + closeChannel(channelFuture.channel()); + } + if (channelToClose != null) { + closeChannel(channelToClose.channel()); + } + } finally { + lock.writeLock().unlock(); + } + } } class InvokeCallbackWrapper implements InvokeCallback { @@ -931,11 +1068,19 @@ public InvokeCallbackWrapper(InvokeCallback invokeCallback, String addr) { @Override public void operationComplete(ResponseFuture responseFuture) { - if (responseFuture != null && responseFuture.isSendRequestOK() && responseFuture.getResponseCommand() != null) { - NettyRemotingClient.this.updateChannelLastResponseTime(addr); - } this.invokeCallback.operationComplete(responseFuture); } + + @Override + public void operationSucceed(RemotingCommand response) { + updateChannelLastResponseTime(addr); + this.invokeCallback.operationSucceed(response); + } + + @Override + public void operationFail(final Throwable throwable) { + this.invokeCallback.operationFail(throwable); + } } class NettyClientHandler extends SimpleChannelInboundHandler { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index 90e358ce3b9..735d36168f4 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -18,6 +18,7 @@ import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.Channel; import io.netty.channel.ChannelDuplexHandler; @@ -52,13 +53,28 @@ import io.netty.util.Timeout; import io.netty.util.TimerTask; import io.netty.util.concurrent.DefaultEventExecutorGroup; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.security.cert.CertificateException; +import java.time.Duration; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.HAProxyConstants; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.BinaryUtil; import org.apache.rocketmq.common.utils.NetworkUtil; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; @@ -71,20 +87,6 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.security.cert.CertificateException; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - @SuppressWarnings("NullableProblems") public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME); @@ -169,7 +171,7 @@ private ExecutorService buildPublicExecutor(NettyServerConfig nettyServerConfig) } private ScheduledExecutorService buildScheduleExecutor() { - return new ScheduledThreadPoolExecutor(1, + return ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("NettyServerScheduler_", true), new ThreadPoolExecutor.DiscardOldestPolicy()); } @@ -303,6 +305,10 @@ private void addCustomConfig(ServerBootstrap childHandler) { @Override public void shutdown() { try { + if (nettyServerConfig.isEnableShutdownGracefully() && isShuttingDown.compareAndSet(false, true)) { + Thread.sleep(Duration.ofSeconds(nettyServerConfig.getShutdownWaitTimeSeconds()).toMillis()); + } + this.timer.stop(); this.eventLoopGroupBoss.shutdownGracefully(); @@ -502,7 +508,7 @@ protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) { case DISABLED: ctx.close(); log.warn("Clients intend to establish an SSL connection while this server is running in SSL disabled mode"); - break; + throw new UnsupportedOperationException("The NettyRemotingServer in SSL disabled mode doesn't support ssl client"); case PERMISSIVE: case ENFORCING: if (null != sslContext) { @@ -734,6 +740,7 @@ public void start() { @Override public void shutdown() { + isShuttingDown.set(true); if (this.serverChannel != null) { try { this.serverChannel.close().await(5, TimeUnit.SECONDS); @@ -787,9 +794,13 @@ private void handleWithMessage(HAProxyMessage msg, Channel channel) { } if (CollectionUtils.isNotEmpty(msg.tlvs())) { msg.tlvs().forEach(tlv -> { + byte[] valueBytes = ByteBufUtil.getBytes(tlv.content()); + if (!BinaryUtil.isAscii(valueBytes)) { + return; + } AttributeKey key = AttributeKeys.valueOf( HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); - String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8)); + String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8)); channel.attr(key).set(value); }); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java index 59ef2c84f15..756661f623f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java @@ -38,6 +38,9 @@ public class NettyServerConfig implements Cloneable { private int serverSocketBacklog = NettySystemConfig.socketBacklog; private boolean serverPooledByteBufAllocatorEnable = true; + private boolean enableShutdownGracefully = false; + private int shutdownWaitTimeSeconds = 30; + /** * make install * @@ -171,4 +174,20 @@ public int getWriteBufferHighWaterMark() { public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) { this.writeBufferHighWaterMark = writeBufferHighWaterMark; } + + public boolean isEnableShutdownGracefully() { + return enableShutdownGracefully; + } + + public void setEnableShutdownGracefully(boolean enableShutdownGracefully) { + this.enableShutdownGracefully = enableShutdownGracefully; + } + + public int getShutdownWaitTimeSeconds() { + return shutdownWaitTimeSeconds; + } + + public void setShutdownWaitTimeSeconds(int shutdownWaitTimeSeconds) { + this.shutdownWaitTimeSeconds = shutdownWaitTimeSeconds; + } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java index 19f705d74bf..0882818feac 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java @@ -22,6 +22,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class ResponseFuture { @@ -59,6 +62,18 @@ public ResponseFuture(Channel channel, int opaque, RemotingCommand request, long public void executeInvokeCallback() { if (invokeCallback != null) { if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) { + RemotingCommand response = getResponseCommand(); + if (response != null) { + invokeCallback.operationSucceed(response); + } else { + if (!isSendRequestOK()) { + invokeCallback.operationFail(new RemotingSendRequestException(channel.remoteAddress().toString(), getCause())); + } else if (isTimeout()) { + invokeCallback.operationFail(new RemotingTimeoutException(channel.remoteAddress().toString(), getTimeoutMillis(), getCause())); + } else { + invokeCallback.operationFail(new RemotingException(getRequestCommand().toString(), getCause())); + } + } invokeCallback.operationComplete(this); } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java index e81dadf2e12..be945c48fda 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java @@ -99,6 +99,8 @@ public class ResponseCode extends RemotingSysResponseCode { public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004; public static final int RPC_TIME_OUT = -1006; + public static final int GO_AWAY = 1500; + /** * Controller response code */ diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java index c937fec2326..d13692735e9 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.remoting.protocol.statictopic; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.rocketmq.common.TopicConfig; @@ -60,4 +61,13 @@ public int hashCode() { .append(mappingDetail) .toHashCode(); } + + @Override + public String toString() { + String string = super.toString(); + if (StringUtils.isNotBlank(string)) { + string = string.substring(0, string.length() - 1) + ", mappingDetail=" + mappingDetail + "]"; + } + return string; + } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java index 133e0ed314e..5328e8845d8 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java @@ -160,31 +160,38 @@ public Promise handlePullMessage(final String addr, RpcRequest rpcR InvokeCallback callback = new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { - RemotingCommand responseCommand = responseFuture.getResponseCommand(); - if (responseCommand == null) { - processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise); - return; - } + + } + + @Override + public void operationSucceed(RemotingCommand response) { try { - switch (responseCommand.getCode()) { + switch (response.getCode()) { case ResponseCode.SUCCESS: case ResponseCode.PULL_NOT_FOUND: case ResponseCode.PULL_RETRY_IMMEDIATELY: case ResponseCode.PULL_OFFSET_MOVED: PullMessageResponseHeader responseHeader = - (PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class); - rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); + (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); + rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody())); default: - RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code")); + RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code")); rpcResponsePromise.setSuccess(rpcResponse); } } catch (Exception e) { - String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand; - RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); + String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + requestCommand; + RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); rpcResponsePromise.setSuccess(rpcResponse); } } + + @Override + public void operationFail(Throwable throwable) { + String errorMessage = "process failed. addr: " + addr + ". Request: " + requestCommand; + RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, throwable)); + rpcResponsePromise.setSuccess(rpcResponse); + } }; this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback); diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java index 90072960b59..d0da0eb2ef7 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java @@ -26,12 +26,12 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; -import org.apache.rocketmq.remoting.netty.ResponseFuture; -import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.NettyClientConfig; -import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.junit.AfterClass; @@ -40,7 +40,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; public class RemotingServerTest { private static RemotingServer remotingServer; @@ -122,10 +121,19 @@ public void testInvokeAsync() throws InterruptedException, RemotingConnectExcept remotingClient.invokeAsync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { latch.countDown(); - assertTrue(responseFuture != null); - assertThat(responseFuture.getResponseCommand().getLanguage()).isEqualTo(LanguageCode.JAVA); - assertThat(responseFuture.getResponseCommand().getExtFields()).hasSize(2); + assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA); + assertThat(response.getExtFields()).hasSize(2); + } + + @Override + public void operationFail(Throwable throwable) { + } }); latch.await(); diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java index de7edbbfba2..a4890d73d5a 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java @@ -144,8 +144,13 @@ else if ("noClientAuthFailure".equals(name.getMethodName())) { tlsClientKeyPath = ""; tlsClientCertPath = ""; clientConfig.setUseTLS(false); - } else if ("serverRejectsSSLClient".equals(name.getMethodName())) { + } else if ("disabledServerRejectsSSLClient".equals(name.getMethodName())) { tlsMode = TlsMode.DISABLED; + } else if ("disabledServerAcceptUnAuthClient".equals(name.getMethodName())) { + tlsMode = TlsMode.DISABLED; + tlsClientKeyPath = ""; + tlsClientCertPath = ""; + clientConfig.setUseTLS(false); } else if ("reloadSslContextForServer".equals(name.getMethodName())) { tlsClientAuthServer = false; tlsServerNeedClientAuth = "none"; @@ -211,7 +216,7 @@ public void serverAcceptsUnAuthClient() throws Exception { } @Test - public void serverRejectsSSLClient() throws Exception { + public void disabledServerRejectsSSLClient() throws Exception { try { RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 5); failBecauseExceptionWasNotThrown(RemotingSendRequestException.class); @@ -219,6 +224,11 @@ public void serverRejectsSSLClient() throws Exception { } } + @Test + public void disabledServerAcceptUnAuthClient() throws Exception { + requestThenAssertResponse(); + } + /** * Tests that a server configured to require client authentication refuses to accept connections * from a client that has an untrusted certificate. diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java similarity index 65% rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java rename to remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java index 6aa5470476f..8ddcdf35df0 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java @@ -14,20 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.remoting.protocol.body; -import java.util.ArrayList; -import java.util.List; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +package org.apache.rocketmq.remoting.netty; -public class GetRemoteClientConfigBody extends RemotingSerializable { - private List keys = new ArrayList<>(); +import io.netty.channel.ChannelFuture; +import io.netty.channel.local.LocalChannel; - public List getKeys() { - return keys; - } - - public void setKeys(List keys) { - this.keys = keys; +public class MockChannel extends LocalChannel { + @Override + public ChannelFuture writeAndFlush(Object msg) { + return new MockChannelPromise(MockChannel.this); } } diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java new file mode 100644 index 00000000000..9c3a354871b --- /dev/null +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.remoting.netty; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelPromise; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GenericFutureListener; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.jetbrains.annotations.NotNull; + +public class MockChannelPromise implements ChannelPromise { + protected Channel channel; + + public MockChannelPromise(Channel channel) { + this.channel = channel; + } + + @Override + public Channel channel() { + return channel; + } + + @Override + public ChannelPromise setSuccess(Void result) { + return this; + } + + @Override + public ChannelPromise setSuccess() { + return this; + } + + @Override + public boolean trySuccess() { + return false; + } + + @Override + public ChannelPromise setFailure(Throwable cause) { + return this; + } + + @Override + public ChannelPromise addListener(GenericFutureListener> listener) { + return this; + } + + @Override + public ChannelPromise addListeners(GenericFutureListener>... listeners) { + return this; + } + + @Override + public ChannelPromise removeListener(GenericFutureListener> listener) { + return this; + } + + @Override + public ChannelPromise removeListeners(GenericFutureListener>... listeners) { + return this; + } + + @Override + public ChannelPromise sync() throws InterruptedException { + return this; + } + + @Override + public ChannelPromise syncUninterruptibly() { + return this; + } + + @Override + public ChannelPromise await() throws InterruptedException { + return this; + } + + @Override + public ChannelPromise awaitUninterruptibly() { + return this; + } + + @Override + public ChannelPromise unvoid() { + return this; + } + + @Override + public boolean isVoid() { + return false; + } + + @Override + public boolean trySuccess(Void result) { + return false; + } + + @Override + public boolean tryFailure(Throwable cause) { + return false; + } + + @Override + public boolean setUncancellable() { + return false; + } + + @Override + public boolean isSuccess() { + return false; + } + + @Override + public boolean isCancellable() { + return false; + } + + @Override + public Throwable cause() { + return null; + } + + @Override + public boolean await(long timeout, TimeUnit unit) throws InterruptedException { + return false; + } + + @Override + public boolean await(long timeoutMillis) throws InterruptedException { + return false; + } + + @Override + public boolean awaitUninterruptibly(long timeout, TimeUnit unit) { + return false; + } + + @Override + public boolean awaitUninterruptibly(long timeoutMillis) { + return false; + } + + @Override + public Void getNow() { + return null; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return false; + } + + @Override + public Void get() throws InterruptedException, ExecutionException { + return null; + } + + @Override + public Void get(long timeout, + @NotNull java.util.concurrent.TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + return null; + } +} diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java index 8381c132b71..dbbea86ea2f 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java @@ -39,9 +39,19 @@ public void testProcessResponseCommand() throws InterruptedException { final Semaphore semaphore = new Semaphore(0); ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { @Override - public void operationComplete(final ResponseFuture responseFuture) { + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { assertThat(semaphore.availablePermits()).isEqualTo(0); } + + @Override + public void operationFail(Throwable throwable) { + + } }, new SemaphoreReleaseOnlyOnce(semaphore)); remotingAbstract.responseTable.putIfAbsent(1, responseFuture); @@ -75,9 +85,19 @@ public void testProcessResponseCommand_RunCallBackInCurrentThread() throws Inter final Semaphore semaphore = new Semaphore(0); ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { @Override - public void operationComplete(final ResponseFuture responseFuture) { + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { assertThat(semaphore.availablePermits()).isEqualTo(0); } + + @Override + public void operationFail(Throwable throwable) { + + } }, new SemaphoreReleaseOnlyOnce(semaphore)); remotingAbstract.responseTable.putIfAbsent(1, responseFuture); @@ -98,7 +118,18 @@ public void testScanResponseTable() { // mock timeout ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() { @Override - public void operationComplete(final ResponseFuture responseFuture) { + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + + } + + @Override + public void operationFail(Throwable throwable) { + } }, null); remotingAbstract.responseTable.putIfAbsent(dummyId, responseFuture); @@ -111,7 +142,22 @@ public void testProcessRequestCommand() throws InterruptedException { final Semaphore semaphore = new Semaphore(0); RemotingCommand request = RemotingCommand.createRequestCommand(1, null); ResponseFuture responseFuture = new ResponseFuture(null, 1, request, 3000, - responseFuture1 -> assertThat(semaphore.availablePermits()).isEqualTo(0), new SemaphoreReleaseOnlyOnce(semaphore)); + new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + assertThat(semaphore.availablePermits()).isEqualTo(0); + } + + @Override + public void operationFail(Throwable throwable) { + + } + }, new SemaphoreReleaseOnlyOnce(semaphore)); remotingAbstract.responseTable.putIfAbsent(1, responseFuture); RemotingCommand response = RemotingCommand.createResponseCommand(0, "Foo"); diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java index 8fabbb21d04..1cc6b4f4687 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java @@ -16,10 +16,17 @@ */ package org.apache.rocketmq.remoting.netty; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.local.LocalChannel; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; import org.apache.rocketmq.remoting.InvokeCallback; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; @@ -29,23 +36,32 @@ import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) public class NettyRemotingClientTest { @Spy private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig()); + @Mock + private RPCHook rpcHookMock; @Test - public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException { + public void testSetCallbackExecutor() { ExecutorService customized = Executors.newCachedThreadPool(); remotingClient.setCallbackExecutor(customized); assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized); @@ -57,13 +73,11 @@ public void testInvokeResponse() throws Exception { RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); - doAnswer(invocation -> { - InvokeCallback callback = invocation.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(response); + CompletableFuture future0 = new CompletableFuture<>(); + future0.complete(responseFuture.getResponseCommand()); + doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); RemotingCommand actual = future.get(); @@ -76,13 +90,9 @@ public void testRemotingSendRequestException() throws Exception { RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); - doAnswer(invocation -> { - InvokeCallback callback = invocation.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - responseFuture.setSendRequestOK(false); - callback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + CompletableFuture future0 = new CompletableFuture<>(); + future0.completeExceptionally(new RemotingSendRequestException(null)); + doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); Throwable thrown = catchThrowable(future::get); @@ -95,12 +105,9 @@ public void testRemotingTimeoutException() throws Exception { RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); - doAnswer(invocation -> { - InvokeCallback callback = invocation.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), -1L, null, null); - callback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + CompletableFuture future0 = new CompletableFuture<>(); + future0.completeExceptionally(new RemotingTimeoutException("")); + doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); Throwable thrown = catchThrowable(future::get); @@ -111,14 +118,9 @@ public void testRemotingTimeoutException() throws Exception { public void testRemotingException() throws Exception { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); - RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - doAnswer(invocation -> { - InvokeCallback callback = invocation.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - callback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + CompletableFuture future0 = new CompletableFuture<>(); + future0.completeExceptionally(new RemotingException("")); + doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); Throwable thrown = catchThrowable(future::get); @@ -134,4 +136,158 @@ public void testInvokeOnewayException() throws Exception { assertThat(e.getMessage()).contains(addr); } } + + @Test + public void testInvoke0() throws ExecutionException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + Channel channel = new MockChannel() { + @Override + public ChannelFuture writeAndFlush(Object msg) { + ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); + responseFuture.setResponseCommand(response); + responseFuture.executeInvokeCallback(); + return super.writeAndFlush(msg); + } + }; + CompletableFuture future = remotingClient.invoke0(channel, request, 1000L); + assertThat(future.get().getResponseCommand()).isEqualTo(response); + } + + @Test + public void testInvoke0WithException() { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + Channel channel = new MockChannel() { + @Override + public ChannelFuture writeAndFlush(Object msg) { + ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); + responseFuture.executeInvokeCallback(); + return super.writeAndFlush(msg); + } + }; + CompletableFuture future = remotingClient.invoke0(channel, request, 1000L); + assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingException.class); + } + + @Test + public void testInvokeSync() throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + remotingClient.registerRPCHook(rpcHookMock); + + Channel channel = new LocalChannel(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); + responseFuture.setResponseCommand(response); + CompletableFuture future = new CompletableFuture<>(); + future.complete(responseFuture); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + RemotingCommand actual = remotingClient.invokeSyncImpl(channel, request, 1000); + assertThat(actual).isEqualTo(response); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); + } + + @Test + public void testInvokeAsync() { + remotingClient.registerRPCHook(rpcHookMock); + Channel channel = new LocalChannel(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); + responseFuture.setResponseCommand(response); + CompletableFuture future = new CompletableFuture<>(); + future.complete(responseFuture); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + + InvokeCallback callback = mock(InvokeCallback.class); + remotingClient.invokeAsyncImpl(channel, request, 1000, callback); + verify(callback, times(1)).operationSucceed(eq(response)); + verify(callback, times(1)).operationComplete(eq(responseFuture)); + verify(callback, never()).operationFail(any()); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); + } + + @Test + public void testInvokeAsyncFail() { + remotingClient.registerRPCHook(rpcHookMock); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + + Channel channel = new LocalChannel(); + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(new RemotingException(null)); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + + InvokeCallback callback = mock(InvokeCallback.class); + remotingClient.invokeAsyncImpl(channel, request, 1000, callback); + verify(callback, never()).operationSucceed(any()); + verify(callback, times(1)).operationComplete(any()); + verify(callback, times(1)).operationFail(any()); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); + } + + @Test + public void testInvokeImpl() throws ExecutionException, InterruptedException { + remotingClient.registerRPCHook(rpcHookMock); + Channel channel = new LocalChannel(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); + responseFuture.setResponseCommand(response); + CompletableFuture future = new CompletableFuture<>(); + future.complete(responseFuture); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + + CompletableFuture future0 = remotingClient.invokeImpl(channel, request, 1000); + assertThat(future0.get()).isEqualTo(responseFuture); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); + } + + @Test + public void testInvokeImplFail() { + remotingClient.registerRPCHook(rpcHookMock); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + + Channel channel = new LocalChannel(); + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(new RemotingException(null)); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + + assertThatThrownBy(() -> remotingClient.invokeImpl(channel, request, 1000).get()).getCause().isInstanceOf(RemotingException.class); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); + } } diff --git a/srvutil/pom.xml b/srvutil/pom.xml index fa54ad01964..894e9cc6fd6 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/store/BUILD.bazel b/store/BUILD.bazel index ba2cd4a02c8..bf594aaa69a 100644 --- a/store/BUILD.bazel +++ b/store/BUILD.bazel @@ -27,6 +27,7 @@ java_library( "@maven//:com_conversantmedia_disruptor", "@maven//:com_google_guava_guava", "@maven//:commons_collections_commons_collections", + "@maven//:commons_io_commons_io", "@maven//:io_netty_netty_all", "@maven//:io_openmessaging_storage_dledger", "@maven//:io_opentelemetry_opentelemetry_api", @@ -40,6 +41,7 @@ java_library( "@maven//:org_apache_commons_commons_lang3", "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", + "@maven//:org_apache_rocketmq_rocketmq_rocksdb", ], ) @@ -60,6 +62,7 @@ java_library( "@maven//:com_google_guava_guava", "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", + "@maven//:org_apache_rocketmq_rocketmq_rocksdb", ], ) @@ -68,6 +71,7 @@ GenTestRules( exclude_tests = [ # This test is extremely slow and flaky, exclude it. "src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest", + "src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest", ], medium_tests = [ "src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest", diff --git a/store/pom.xml b/store/pom.xml index 38f04009db5..e1e61612354 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 @@ -58,6 +58,10 @@ com.google.guava guava + + commons-io + commons-io + org.slf4j diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java index dca7d53258d..c8420fea11f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java +++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java @@ -55,7 +55,7 @@ public MappedFile putRequestAndReturnMappedFile(String nextFilePath, String next if (this.messageStore.isTransientStorePoolEnable()) { if (this.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool() && BrokerRole.SLAVE != this.messageStore.getMessageStoreConfig().getBrokerRole()) { //if broker is slave, don't fast fail even no buffer in pool - canSubmitRequests = this.messageStore.getTransientStorePool().availableBufferNums() - this.requestQueue.size(); + canSubmitRequests = this.messageStore.remainTransientStoreBufferNumbs() - this.requestQueue.size(); } } @@ -65,7 +65,7 @@ public MappedFile putRequestAndReturnMappedFile(String nextFilePath, String next if (nextPutOK) { if (canSubmitRequests <= 0) { log.warn("[NOTIFYME]TransientStorePool is not enough, so create mapped file error, " + - "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums()); + "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs()); this.requestTable.remove(nextFilePath); return null; } @@ -81,7 +81,7 @@ public MappedFile putRequestAndReturnMappedFile(String nextFilePath, String next if (nextNextPutOK) { if (canSubmitRequests <= 0) { log.warn("[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, " + - "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums()); + "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs()); this.requestTable.remove(nextNextFilePath); } else { boolean offerOK = this.requestQueue.offer(nextNextReq); diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index e6ee3bacc15..93102799b7c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,6 +60,8 @@ import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.util.LibC; +import org.rocksdb.RocksDBException; + import sun.nio.ch.DirectBuffer; /** @@ -122,7 +124,7 @@ protected PutMessageThreadLocal initialValue() { this.flushDiskWatcher = new FlushDiskWatcher(); - this.topicQueueLock = new TopicQueueLock(); + this.topicQueueLock = new TopicQueueLock(messageStore.getMessageStoreConfig().getTopicQueueLockNum()); this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); } @@ -299,8 +301,9 @@ public boolean getLastMappedFile(final long startOffset) { /** * When the normal exit, data recovery, all memory data have been flush + * @throws RocksDBException only in rocksdb mode */ - public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { + public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); final List mappedFiles = this.mappedFileQueue.getMappedFiles(); @@ -369,21 +372,22 @@ else if (!dispatchRequest.isSuccess()) { this.setConfirmOffset(lastValidMsgPhyOffset); } - this.mappedFileQueue.setFlushedWhere(processOffset); - this.mappedFileQueue.setCommittedWhere(processOffset); - this.mappedFileQueue.truncateDirtyFiles(processOffset); - // Clear ConsumeQueue redundant data if (maxPhyOffsetOfConsumeQueue >= processOffset) { log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); } + + this.mappedFileQueue.setFlushedWhere(processOffset); + this.mappedFileQueue.setCommittedWhere(processOffset); + this.mappedFileQueue.truncateDirtyFiles(processOffset); } else { // Commitlog case files are deleted log.warn("The commitlog files are deleted, and delete the consume queue files"); this.mappedFileQueue.setFlushedWhere(0); this.mappedFileQueue.setCommittedWhere(0); - this.defaultMessageStore.destroyLogics(); + this.defaultMessageStore.getQueueStore().destroy(); + this.defaultMessageStore.getQueueStore().loadAfterDestroy(); } } @@ -580,7 +584,7 @@ public long getConfirmOffset() { return this.defaultMessageStore.getMaxPhyOffset(); } // First time it will compute the confirmOffset. - if (this.confirmOffset <= 0) { + if (this.confirmOffset < 0) { setConfirmOffset(((AutoSwitchHAService) this.defaultMessageStore.getHaService()).computeConfirmOffset()); log.info("Init the confirmOffset to {}.", this.confirmOffset); } @@ -626,8 +630,10 @@ public long getLastFileFromOffset() { return -1; } - @Deprecated - public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { + /** + * @throws RocksDBException only in rocksdb mode + */ + public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { // recover by the minimum time stamp boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); @@ -705,6 +711,9 @@ else if (size == 0) { } } + // only for rocksdb mode + this.getMessageStore().finishCommitLogDispatch(); + processOffset += mappedFileOffset; if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) { if (this.defaultMessageStore.getConfirmOffset() < this.defaultMessageStore.getMinPhyOffset()) { @@ -717,22 +726,24 @@ else if (size == 0) { } else { this.setConfirmOffset(lastValidMsgPhyOffset); } - this.mappedFileQueue.setFlushedWhere(processOffset); - this.mappedFileQueue.setCommittedWhere(processOffset); - this.mappedFileQueue.truncateDirtyFiles(processOffset); // Clear ConsumeQueue redundant data if (maxPhyOffsetOfConsumeQueue >= processOffset) { log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); } + + this.mappedFileQueue.setFlushedWhere(processOffset); + this.mappedFileQueue.setCommittedWhere(processOffset); + this.mappedFileQueue.truncateDirtyFiles(processOffset); } // Commitlog case files are deleted else { log.warn("The commitlog files are deleted, and delete the consume queue files"); this.mappedFileQueue.setFlushedWhere(0); this.mappedFileQueue.setCommittedWhere(0); - this.defaultMessageStore.destroyLogics(); + this.defaultMessageStore.getQueueStore().destroy(); + this.defaultMessageStore.getQueueStore().loadAfterDestroy(); } } @@ -755,7 +766,7 @@ protected void onCommitLogAppend(MessageExtBrokerInner msg, AppendMessageResult this.getMessageStore().onCommitLogAppend(msg, result, commitLogFile); } - private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) { + private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) throws RocksDBException { ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); int magicCode = byteBuffer.getInt(MessageDecoder.MESSAGE_MAGIC_CODE_POSITION); @@ -763,28 +774,37 @@ private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) { return false; } - int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); - int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; - int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornhostLength; - long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); - if (0 == storeTimestamp) { - return false; - } - - if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() - && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { - if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { - log.info("find check timestamp, {} {}", - storeTimestamp, - UtilAll.timeMillisToHumanString(storeTimestamp)); + if (this.defaultMessageStore.getMessageStoreConfig().isEnableRocksDBStore()) { + final long maxPhyOffsetInConsumeQueue = this.defaultMessageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(); + long phyOffset = byteBuffer.getLong(MessageDecoder.MESSAGE_PHYSIC_OFFSET_POSITION); + if (phyOffset <= maxPhyOffsetInConsumeQueue) { + log.info("find check. beginPhyOffset: {}, maxPhyOffsetInConsumeQueue: {}", phyOffset, maxPhyOffsetInConsumeQueue); return true; } } else { - if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { - log.info("find check timestamp, {} {}", - storeTimestamp, - UtilAll.timeMillisToHumanString(storeTimestamp)); - return true; + int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); + int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornHostLength; + long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); + if (0 == storeTimestamp) { + return false; + } + + if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() + && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { + if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { + log.info("find check timestamp, {} {}", + storeTimestamp, + UtilAll.timeMillisToHumanString(storeTimestamp)); + return true; + } + } else { + if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { + log.info("find check timestamp, {} {}", + storeTimestamp, + UtilAll.timeMillisToHumanString(storeTimestamp)); + return true; + } } } @@ -958,8 +978,6 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke beginTimeInLock = 0; return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result)); case UNKNOWN_ERROR: - beginTimeInLock = 0; - return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); default: beginTimeInLock = 0; return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); @@ -974,6 +992,8 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) { this.defaultMessageStore.increaseOffset(msg, getMessageNum(msg)); } + } catch (RocksDBException e) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); } finally { topicQueueLock.unlock(topicQueueKey); } @@ -997,7 +1017,7 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke public CompletableFuture asyncPutMessages(final MessageExtBatch messageExtBatch) { messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); - AppendMessageResult result; + AppendMessageResult result = null; StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); @@ -1133,7 +1153,9 @@ public CompletableFuture asyncPutMessages(final MessageExtBatc if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) { this.defaultMessageStore.increaseOffset(messageExtBatch, (short) putMessageContext.getBatchSize()); } - } finally { + } catch (RocksDBException e) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); + } finally { topicQueueLock.unlock(topicQueueKey); } diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java index 9d6fa6ad98b..f3a7b7c5c43 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java @@ -17,6 +17,8 @@ package org.apache.rocketmq.store; +import org.rocksdb.RocksDBException; + /** * Dispatcher of commit log. */ @@ -25,6 +27,7 @@ public interface CommitLogDispatcher { /** * Dispatch messages from store to build consume queues, indexes, and filter data * @param request dispatch message request + * @throws RocksDBException only in rocksdb mode */ - void dispatch(final DispatchRequest request); + void dispatch(final DispatchRequest request) throws RocksDBException; } diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index a0b886eb0e9..623509c8bf2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -24,13 +24,12 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.attribute.CQType; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.config.BrokerRole; @@ -39,9 +38,9 @@ import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.FileQueueLifeCycle; +import org.apache.rocketmq.store.queue.MultiDispatch; import org.apache.rocketmq.store.queue.QueueOffsetOperator; import org.apache.rocketmq.store.queue.ReferredIterator; -import org.apache.rocketmq.store.timer.TimerMessageStore; public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); @@ -145,7 +144,7 @@ public void recover() { if (offset >= 0 && size > 0) { mappedFileOffset = i + CQ_STORE_UNIT_SIZE; - this.maxPhysicOffset = offset + size; + this.setMaxPhysicOffset(offset + size); if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; } @@ -409,7 +408,7 @@ public void truncateDirtyLogicFiles(long phyOffset, boolean deleteFile) { int logicFileSize = this.mappedFileSize; - this.maxPhysicOffset = phyOffset; + this.setMaxPhysicOffset(phyOffset); long maxExtAddr = 1; boolean shouldDeleteFile = false; while (true) { @@ -435,7 +434,7 @@ public void truncateDirtyLogicFiles(long phyOffset, boolean deleteFile) { mappedFile.setWrotePosition(pos); mappedFile.setCommittedPosition(pos); mappedFile.setFlushedPosition(pos); - this.maxPhysicOffset = offset + size; + this.setMaxPhysicOffset(offset + size); // This maybe not take effect, when not every consume queue has extend file. if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; @@ -453,7 +452,7 @@ public void truncateDirtyLogicFiles(long phyOffset, boolean deleteFile) { mappedFile.setWrotePosition(pos); mappedFile.setCommittedPosition(pos); mappedFile.setFlushedPosition(pos); - this.maxPhysicOffset = offset + size; + this.setMaxPhysicOffset(offset + size); if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; } @@ -703,7 +702,7 @@ public void putMessagePositionInfoWrapper(DispatchRequest request) { this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp()); } this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp()); - if (checkMultiDispatchQueue(request)) { + if (MultiDispatch.checkMultiDispatchQueue(this.messageStore.getMessageStoreConfig(), request)) { multiDispatchLmqQueue(request, maxRetries); } return; @@ -725,25 +724,6 @@ public void putMessagePositionInfoWrapper(DispatchRequest request) { this.messageStore.getRunningFlags().makeLogicsQueueError(); } - private boolean checkMultiDispatchQueue(DispatchRequest dispatchRequest) { - if (!this.messageStore.getMessageStoreConfig().isEnableMultiDispatch() - || dispatchRequest.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) - || dispatchRequest.getTopic().equals(TimerMessageStore.TIMER_TOPIC) - || dispatchRequest.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC)) { - return false; - } - Map prop = dispatchRequest.getPropertiesMap(); - if (prop == null || prop.isEmpty()) { - return false; - } - String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); - String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); - if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { - return false; - } - return true; - } - private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) { Map prop = request.getPropertiesMap(); String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); @@ -765,9 +745,7 @@ private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) { queueId = 0; } doDispatchLmqQueue(request, maxRetries, queueName, queueOffset, queueId); - } - return; } private void doDispatchLmqQueue(DispatchRequest request, int maxRetries, String queueName, long queueOffset, @@ -802,7 +780,7 @@ public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageEx // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. - if (!isNeedHandleMultiDispatch(msg)) { + if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { return; } String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); @@ -812,14 +790,14 @@ public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageEx String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); Long[] queueOffsets = new Long[queues.length]; for (int i = 0; i < queues.length; i++) { - String key = queueKey(queues[i], msg); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { + String key = MultiDispatch.lmqQueueKey(queues[i]); queueOffsets[i] = queueOffsetOperator.getLmqOffset(key); } } MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); - removeWaitStorePropertyString(msg); + msg.removeWaitStorePropertyString(); } @Override @@ -830,7 +808,7 @@ public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, Message // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. - if (!isNeedHandleMultiDispatch(msg)) { + if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { return; } String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); @@ -839,50 +817,18 @@ public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, Message } String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); for (int i = 0; i < queues.length; i++) { - String key = queueKey(queues[i], msg); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { + String key = MultiDispatch.lmqQueueKey(queues[i]); queueOffsetOperator.increaseLmqOffset(key, (short) 1); } } } - public boolean isNeedHandleMultiDispatch(MessageExtBrokerInner msg) { - return messageStore.getMessageStoreConfig().isEnableMultiDispatch() - && !msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) - && !msg.getTopic().equals(TimerMessageStore.TIMER_TOPIC) - && !msg.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); - } - - public String queueKey(String queueName, MessageExtBrokerInner msgInner) { - StringBuilder keyBuilder = new StringBuilder(); - keyBuilder.append(queueName); - keyBuilder.append('-'); - int queueId = msgInner.getQueueId(); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { - queueId = 0; - } - keyBuilder.append(queueId); - return keyBuilder.toString(); - } - - private void removeWaitStorePropertyString(MessageExtBrokerInner msgInner) { - if (msgInner.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) { - // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message. - // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it. - String waitStoreMsgOKValue = msgInner.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); - msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later - msgInner.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue); - } else { - msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - } - } - private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, final long cqOffset) { - if (offset + size <= this.maxPhysicOffset) { - log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset); + if (offset + size <= this.getMaxPhysicOffset()) { + log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset); return true; } @@ -926,7 +872,7 @@ private boolean putMessagePositionInfo(final long offset, final int size, final ); } } - this.maxPhysicOffset = offset + size; + this.setMaxPhysicOffset(offset + size); return mappedFile.appendMessage(this.byteBufferIndex.array()); } return false; @@ -965,6 +911,11 @@ public ReferredIterator iterateFrom(long startOffset) { return new ConsumeQueueIterator(sbr); } + @Override + public ReferredIterator iterateFrom(long startIndex, int count) { + return iterateFrom(startIndex); + } + @Override public CqUnit get(long offset) { ReferredIterator it = iterateFrom(offset); @@ -974,6 +925,20 @@ public CqUnit get(long offset) { return it.nextAndRelease(); } + @Override + public Pair getCqUnitAndStoreTime(long index) { + CqUnit cqUnit = get(index); + Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); + return new Pair<>(cqUnit, messageStoreTime); + } + + @Override + public Pair getEarliestUnitAndStoreTime() { + CqUnit cqUnit = getEarliestUnit(); + Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); + return new Pair<>(cqUnit, messageStoreTime); + } + @Override public CqUnit getEarliestUnit() { /** @@ -1130,7 +1095,7 @@ public void setMaxPhysicOffset(long maxPhysicOffset) { @Override public void destroy() { - this.maxPhysicOffset = -1; + this.setMaxPhysicOffset(-1); this.minLogicOffset = 0; this.mappedFileQueue.destroy(); if (isExtReadEnable()) { diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 6115ead598f..99a54e2d7ff 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -48,7 +48,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; @@ -83,6 +82,7 @@ import org.apache.rocketmq.common.utils.CleanupPolicyUtils; import org.apache.rocketmq.common.utils.QueueTypeUtils; import org.apache.rocketmq.common.utils.ServiceProvider; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -105,32 +105,35 @@ import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.ConsumeQueueStore; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.PerfCounter; +import org.rocksdb.RocksDBException; public class DefaultMessageStore implements MessageStore { - private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + protected static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); public final PerfCounter.Ticks perfs = new PerfCounter.Ticks(LOGGER); private final MessageStoreConfig messageStoreConfig; // CommitLog - private final CommitLog commitLog; + protected final CommitLog commitLog; - private final ConsumeQueueStore consumeQueueStore; + protected final ConsumeQueueStoreInterface consumeQueueStore; private final FlushConsumeQueueService flushConsumeQueueService; - private final CleanCommitLogService cleanCommitLogService; + protected final CleanCommitLogService cleanCommitLogService; private final CleanConsumeQueueService cleanConsumeQueueService; private final CorrectLogicOffsetService correctLogicOffsetService; - private final IndexService indexService; + protected final IndexService indexService; private final AllocateMappedFileService allocateMappedFileService; @@ -147,7 +150,7 @@ public class DefaultMessageStore implements MessageStore { private final TransientStorePool transientStorePool; - private final RunningFlags runningFlags = new RunningFlags(); + protected final RunningFlags runningFlags = new RunningFlags(); private final SystemClock systemClock = new SystemClock(); private final ScheduledExecutorService scheduledExecutorService; @@ -156,6 +159,7 @@ public class DefaultMessageStore implements MessageStore { private final BrokerConfig brokerConfig; private volatile boolean shutdown = true; + protected boolean notifyMessageArriveInBatch = false; private StoreCheckpoint storeCheckpoint; private TimerMessageStore timerMessageStore; @@ -182,7 +186,7 @@ public class DefaultMessageStore implements MessageStore { private volatile long brokerInitMaxOffset = -1L; - protected List putMessageHookList = new ArrayList<>(); + private List putMessageHookList = new ArrayList<>(); private SendMessageBackHook sendMessageBackHook; @@ -205,7 +209,7 @@ public class DefaultMessageStore implements MessageStore { private ConcurrentMap topicConfigTable; private final ScheduledExecutorService scheduledCleanQueueExecutorService = - Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); + ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws IOException { @@ -222,12 +226,12 @@ public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final Br this.commitLog = new CommitLog(this); } - this.consumeQueueStore = new ConsumeQueueStore(this, this.messageStoreConfig); + this.consumeQueueStore = createConsumeQueueStore(); - this.flushConsumeQueueService = new FlushConsumeQueueService(); + this.flushConsumeQueueService = createFlushConsumeQueueService(); this.cleanCommitLogService = new CleanCommitLogService(); - this.cleanConsumeQueueService = new CleanConsumeQueueService(); - this.correctLogicOffsetService = new CorrectLogicOffsetService(); + this.cleanConsumeQueueService = createCleanConsumeQueueService(); + this.correctLogicOffsetService = createCorrectLogicOffsetService(); this.storeStatsService = new StoreStatsService(getBrokerIdentity()); this.indexService = new IndexService(this); @@ -250,10 +254,10 @@ public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final Br this.reputMessageService = new ConcurrentReputMessageService(); } - this.transientStorePool = new TransientStorePool(this); + this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog()); this.scheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); + ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); this.dispatcherList = new LinkedList<>(); this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue()); @@ -273,6 +277,22 @@ public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final Br parseDelayLevel(); } + public ConsumeQueueStoreInterface createConsumeQueueStore() { + return new ConsumeQueueStore(this); + } + + public CleanConsumeQueueService createCleanConsumeQueueService() { + return new CleanConsumeQueueService(); + } + + public FlushConsumeQueueService createFlushConsumeQueueService() { + return new FlushConsumeQueueService(); + } + + public CorrectLogicOffsetService createCorrectLogicOffsetService() { + return new CorrectLogicOffsetService(); + } + public boolean parseDelayLevel() { HashMap timeUnitTable = new HashMap<>(); timeUnitTable.put("s", 1000L); @@ -305,7 +325,7 @@ public boolean parseDelayLevel() { } @Override - public void truncateDirtyLogicFiles(long phyOffset) { + public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException { this.consumeQueueStore.truncateDirty(phyOffset); } @@ -393,6 +413,7 @@ public void start() throws Exception { this.flushConsumeQueueService.start(); this.commitLog.start(); + this.consumeQueueStore.start(); this.storeStatsService.start(); if (this.haService != null) { @@ -481,6 +502,7 @@ public void shutdown() { this.storeStatsService.shutdown(); this.commitLog.shutdown(); this.reputMessageService.shutdown(); + this.consumeQueueStore.shutdown(); // dispatch-related services must be shut down after reputMessageService this.indexService.shutdown(); if (this.compactionService != null) { @@ -515,7 +537,7 @@ public void shutdown() { @Override public void destroy() { - this.destroyLogics(); + this.consumeQueueStore.destroy(); this.commitLog.destroy(); this.indexService.destroy(); this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); @@ -541,11 +563,6 @@ public long getMajorFileSize() { return commitLogSize + consumeQueueSize + indexFileSize; } - @Override - public void destroyLogics() { - this.consumeQueueStore.destroy(); - } - @Override public CompletableFuture asyncPutMessage(MessageExtBrokerInner msg) { @@ -687,7 +704,7 @@ public CommitLog getCommitLog() { return commitLog; } - public void truncateDirtyFiles(long offsetToTruncate) { + public void truncateDirtyFiles(long offsetToTruncate) throws RocksDBException { LOGGER.info("truncate dirty files to {}", offsetToTruncate); @@ -700,12 +717,12 @@ public void truncateDirtyFiles(long offsetToTruncate) { long oldReputFromOffset = this.reputMessageService.getReputFromOffset(); - // truncate commitLog - this.commitLog.truncateDirtyFiles(offsetToTruncate); - // truncate consume queue this.truncateDirtyLogicFiles(offsetToTruncate); + // truncate commitLog + this.commitLog.truncateDirtyFiles(offsetToTruncate); + this.recoverTopicQueueTable(); if (!messageStoreConfig.isEnableBuildConsumeQueueConcurrently()) { @@ -723,7 +740,7 @@ public void truncateDirtyFiles(long offsetToTruncate) { } @Override - public boolean truncateFiles(long offsetToTruncate) { + public boolean truncateFiles(long offsetToTruncate) throws RocksDBException { if (offsetToTruncate >= this.getMaxPhyOffset()) { LOGGER.info("no need to truncate files, truncate offset is {}, max physical offset is {}", offsetToTruncate, this.getMaxPhyOffset()); return true; @@ -825,17 +842,19 @@ public GetMessageResult getMessage(final String group, final String topic, final while (getResult.getBufferTotalSize() <= 0 && nextBeginOffset < maxOffset && cqFileNum++ < this.messageStoreConfig.getTravelCqFileNumWhenGetMessage()) { - ReferredIterator bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset); - - if (bufferConsumeQueue == null) { - status = GetMessageStatus.OFFSET_FOUND_NULL; - nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset)); - LOGGER.warn("consumer request topic: " + topic + "offset: " + offset + " minOffset: " + minOffset + " maxOffset: " - + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset); - break; - } + ReferredIterator bufferConsumeQueue = null; try { + bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset, maxMsgNums); + + if (bufferConsumeQueue == null) { + status = GetMessageStatus.OFFSET_FOUND_NULL; + nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset)); + LOGGER.warn("consumer request topic: " + topic + ", offset: " + offset + ", minOffset: " + minOffset + ", maxOffset: " + + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset); + break; + } + long nextPhyFileStartOffset = Long.MIN_VALUE; while (bufferConsumeQueue.hasNext() && nextBeginOffset < maxOffset) { @@ -905,8 +924,13 @@ public GetMessageResult getMessage(final String group, final String topic, final status = GetMessageStatus.FOUND; nextPhyFileStartOffset = Long.MIN_VALUE; } + } catch (RocksDBException e) { + ERROR_LOG.error("getMessage Failed. cid: {}, topic: {}, queueId: {}, offset: {}, minOffset: {}, maxOffset: {}, {}", + group, topic, queueId, offset, minOffset, maxOffset, e.getMessage()); } finally { - bufferConsumeQueue.release(); + if (bufferConsumeQueue != null) { + bufferConsumeQueue.release(); + } } } @@ -975,12 +999,12 @@ public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { @Override public long getMinOffsetInQueue(String topic, int queueId) { - ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); - if (logic != null) { - return logic.getMinOffsetInQueue(); + try { + return this.consumeQueueStore.getMinOffsetInQueue(topic, queueId); + } catch (RocksDBException e) { + ERROR_LOG.error("getMinOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e); + return -1; } - - return -1; } @Override @@ -997,38 +1021,27 @@ public void setTimerMessageStore(TimerMessageStore timerMessageStore) { public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) { ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); if (consumeQueue != null) { - - ReferredIterator bufferConsumeQueue = consumeQueue.iterateFrom(consumeQueueOffset); - if (bufferConsumeQueue != null) { - try { - if (bufferConsumeQueue.hasNext()) { - long offsetPy = bufferConsumeQueue.next().getPos(); - return offsetPy; - } - } finally { - bufferConsumeQueue.release(); - } + CqUnit cqUnit = consumeQueue.get(consumeQueueOffset); + if (cqUnit != null) { + return cqUnit.getPos(); } } - return 0; } @Override public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) { - return getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER); + return this.getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER); } + @Override public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) { - ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); - if (logic != null) { - long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType); - // Make sure the result offset is in valid range. - resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue()); - resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue()); - return resultOffset; + try { + return this.consumeQueueStore.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType); + } catch (RocksDBException e) { + ERROR_LOG.error("getOffsetInQueueByTime Failed. topic: {}, queueId: {}, timestamp: {} boundaryType: {}, {}", + topic, queueId, timestamp, boundaryType, e.getMessage()); } - return 0; } @@ -1088,6 +1101,10 @@ public String getStorePathLogic() { return StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()); } + public MessageArrivingListener getMessageArrivingListener() { + return messageArrivingListener; + } + @Override public HashMap getRuntimeInfo() { HashMap result = this.storeStatsService.getRuntimeInfo(); @@ -1121,7 +1138,6 @@ public long getMaxPhyOffset() { return this.commitLog.getMaxOffset(); } - @Override public long getMinPhyOffset() { return this.commitLog.getMinOffset(); @@ -1141,7 +1157,10 @@ public boolean getLastMappedFile(long startOffset) { public long getEarliestMessageTime(String topic, int queueId) { ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); if (logicQueue != null) { - return getStoreTime(logicQueue.getEarliestUnit()); + Pair pair = logicQueue.getEarliestUnitAndStoreTime(); + if (pair != null && pair.getObject2() != null) { + return pair.getObject2(); + } } return -1; @@ -1152,19 +1171,6 @@ public CompletableFuture getEarliestMessageTimeAsync(String topic, int que return CompletableFuture.completedFuture(getEarliestMessageTime(topic, queueId)); } - protected long getStoreTime(CqUnit result) { - if (result != null) { - try { - final long phyOffset = result.getPos(); - final int size = result.getSize(); - long storeTime = this.getCommitLog().pickupStoreTimestamp(phyOffset, size); - return storeTime; - } catch (Exception e) { - } - } - return -1; - } - @Override public long getEarliestMessageTime() { long minPhyOffset = this.getMinPhyOffset(); @@ -1179,13 +1185,16 @@ public long getEarliestMessageTime() { public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) { ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); if (logicQueue != null) { - return getStoreTime(logicQueue.get(consumeQueueOffset)); + Pair pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset); + if (pair != null && pair.getObject2() != null) { + return pair.getObject2(); + } } - return -1; } - @Override public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, + @Override + public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, long consumeQueueOffset) { return CompletableFuture.completedFuture(getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset)); } @@ -1354,6 +1363,7 @@ public long now() { * If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned, * consume queue will be created with old offset, then later message with new offset table can not be * dispatched to consume queue. + * @throws RocksDBException only in rocksdb mode */ @Override public int deleteTopics(final Set deleteTopics) { @@ -1363,17 +1373,19 @@ public int deleteTopics(final Set deleteTopics) { int deleteCount = 0; for (String topic : deleteTopics) { - ConcurrentMap queueTable = - this.consumeQueueStore.getConsumeQueueTable().get(topic); + ConcurrentMap queueTable = this.consumeQueueStore.findConsumeQueueMap(topic); if (queueTable == null || queueTable.isEmpty()) { continue; } for (ConsumeQueueInterface cq : queueTable.values()) { - this.consumeQueueStore.destroy(cq); - LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", - cq.getTopic(), cq.getQueueId()); + try { + this.consumeQueueStore.destroy(cq); + } catch (RocksDBException e) { + LOGGER.error("DeleteTopic: ConsumeQueue cleans error!, topic={}, queueId={}", cq.getTopic(), cq.getQueueId(), e); + } + LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", cq.getTopic(), cq.getQueueId()); this.consumeQueueStore.removeTopicQueueTable(cq.getTopic(), cq.getQueueId()); } @@ -1852,14 +1864,18 @@ private boolean isTempFileExist() { return file.exists(); } - private void recover(final boolean lastExitOK) { - boolean recoverConcurrently = this.brokerConfig.isRecoverConcurrently(); + private boolean isRecoverConcurrently() { + return this.brokerConfig.isRecoverConcurrently() && !this.messageStoreConfig.isEnableRocksDBStore(); + } + + private void recover(final boolean lastExitOK) throws RocksDBException { + boolean recoverConcurrently = this.isRecoverConcurrently(); LOGGER.info("message store recover mode: {}", recoverConcurrently ? "concurrent" : "normal"); // recover consume queue long recoverConsumeQueueStart = System.currentTimeMillis(); this.recoverConsumeQueue(); - long maxPhyOffsetOfConsumeQueue = this.getMaxOffsetInConsumeQueue(); + long maxPhyOffsetOfConsumeQueue = this.consumeQueueStore.getMaxPhyOffsetInConsumeQueue(); long recoverConsumeQueueEnd = System.currentTimeMillis(); // recover commitlog @@ -1894,23 +1910,25 @@ public MessageStoreConfig getMessageStoreConfig() { return messageStoreConfig; } + @Override + public void finishCommitLogDispatch() { + // ignore + } + @Override public TransientStorePool getTransientStorePool() { return transientStorePool; } private void recoverConsumeQueue() { - if (!this.brokerConfig.isRecoverConcurrently()) { + if (!this.isRecoverConcurrently()) { this.consumeQueueStore.recover(); } else { this.consumeQueueStore.recoverConcurrently(); } } - private long getMaxOffsetInConsumeQueue() { - return this.consumeQueueStore.getMaxOffsetInConsumeQueue(); - } - + @Override public void recoverTopicQueueTable() { long minPhyOffset = this.commitLog.getMinOffset(); this.consumeQueueStore.recoverOffsetTable(minPhyOffset); @@ -1949,13 +1967,17 @@ public RunningFlags getRunningFlags() { return runningFlags; } - public void doDispatch(DispatchRequest req) { + public void doDispatch(DispatchRequest req) throws RocksDBException { for (CommitLogDispatcher dispatcher : this.dispatcherList) { dispatcher.dispatch(req); } } - public void putMessagePositionInfo(DispatchRequest dispatchRequest) { + /** + * @param dispatchRequest + * @throws RocksDBException only in rocksdb mode + */ + protected void putMessagePositionInfo(DispatchRequest dispatchRequest) throws RocksDBException { this.consumeQueueStore.putMessagePositionInfoWrapper(dispatchRequest); } @@ -1983,7 +2005,10 @@ public BrokerConfig getBrokerConfig() { } public int remainTransientStoreBufferNumbs() { - return this.transientStorePool.availableBufferNums(); + if (this.isTransientStorePoolEnable()) { + return this.transientStorePool.availableBufferNums(); + } + return Integer.MAX_VALUE; } @Override @@ -2051,7 +2076,7 @@ public PerfCounter.Ticks getPerfCounter() { } @Override - public ConsumeQueueStore getQueueStore() { + public ConsumeQueueStoreInterface getQueueStore() { return consumeQueueStore; } @@ -2062,7 +2087,7 @@ public void onCommitLogAppend(MessageExtBrokerInner msg, AppendMessageResult res @Override public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, - boolean isRecover, boolean isFileEnd) { + boolean isRecover, boolean isFileEnd) throws RocksDBException { if (doDispatch && !isFileEnd) { this.doDispatch(dispatchRequest); } @@ -2079,7 +2104,7 @@ public boolean isSyncMaster() { } @Override - public void assignOffset(MessageExtBrokerInner msg) { + public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException { final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { @@ -2124,12 +2149,12 @@ public BrokerIdentity getBrokerIdentity() { class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher { @Override - public void dispatch(DispatchRequest request) { + public void dispatch(DispatchRequest request) throws RocksDBException { final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag()); switch (tranType) { case MessageSysFlag.TRANSACTION_NOT_TYPE: case MessageSysFlag.TRANSACTION_COMMIT_TYPE: - DefaultMessageStore.this.putMessagePositionInfo(request); + putMessagePositionInfo(request); break; case MessageSysFlag.TRANSACTION_PREPARED_TYPE: case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: @@ -2275,7 +2300,7 @@ public String getServiceName() { return DefaultMessageStore.this.brokerConfig.getIdentifier() + CleanCommitLogService.class.getSimpleName(); } - private boolean isTimeToDelete() { + protected boolean isTimeToDelete() { String when = DefaultMessageStore.this.getMessageStoreConfig().getDeleteWhen(); if (UtilAll.isItTimeToDo(when)) { DefaultMessageStore.LOGGER.info("it's time to reclaim disk space, " + when); @@ -2433,7 +2458,7 @@ public boolean isSpaceFull() { } class CleanConsumeQueueService { - private long lastPhysicalMinOffset = 0; + protected long lastPhysicalMinOffset = 0; public void run() { try { @@ -2443,7 +2468,7 @@ public void run() { } } - private void deleteExpiredFiles() { + protected void deleteExpiredFiles() { int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval(); long minOffset = DefaultMessageStore.this.commitLog.getMinOffset(); @@ -2548,7 +2573,7 @@ private boolean needCorrect(ConsumeQueueInterface logic, long minPhyOffset, long if (cqUnit.getPos() >= minPhyOffset) { - // Normal case, do not need correct. + // Normal case, do not need to correct. return false; } } @@ -2738,6 +2763,18 @@ public synchronized boolean isEmpty() { } + @Override + public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { + if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() + && DefaultMessageStore.this.messageArrivingListener != null) { + DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), + dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, + dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), + dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); + DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest); + } + } + class ReputMessageService extends ServiceThread { protected volatile long reputFromOffset = 0; @@ -2807,13 +2844,8 @@ public void doReput() { if (size > 0) { DefaultMessageStore.this.doDispatch(dispatchRequest); - if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() - && DefaultMessageStore.this.messageArrivingListener != null) { - DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), - dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, - dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), - dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); - notifyMessageArrive4MultiQueue(dispatchRequest); + if (!notifyMessageArriveInBatch) { + notifyMessageArriveIfNecessary(dispatchRequest); } this.reputFromOffset += size; @@ -2847,9 +2879,14 @@ public void doReput() { } } } + } catch (RocksDBException e) { + ERROR_LOG.info("dispatch message to cq exception. reputFromOffset: {}", this.reputFromOffset, e); + return; } finally { result.release(); } + + finishCommitLogDispatch(); } } @@ -2912,7 +2949,7 @@ class MainBatchDispatchRequestService extends ServiceThread { private final ExecutorService batchDispatchRequestExecutor; public MainBatchDispatchRequestService() { - batchDispatchRequestExecutor = new ThreadPoolExecutor( + batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor( DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), 1000 * 60, @@ -2986,7 +3023,7 @@ class DispatchService extends ServiceThread { // dispatchRequestsList:[ // {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}, // {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}] - private void dispatch() { + private void dispatch() throws Exception { dispatchRequestsList.clear(); dispatchRequestOrderlyQueue.get(dispatchRequestsList); if (!dispatchRequestsList.isEmpty()) { @@ -2994,21 +3031,15 @@ private void dispatch() { for (DispatchRequest dispatchRequest : dispatchRequests) { DefaultMessageStore.this.doDispatch(dispatchRequest); // wake up long-polling - if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() - && DefaultMessageStore.this.messageArrivingListener != null) { - DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), - dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, - dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), - dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); - DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest); - } + DefaultMessageStore.this.notifyMessageArriveIfNecessary(dispatchRequest); + if (!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() && - DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { + DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { DefaultMessageStore.this.storeStatsService - .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1); + .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1); DefaultMessageStore.this.storeStatsService - .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) - .add(dispatchRequest.getMsgSize()); + .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) + .add(dispatchRequest.getMsgSize()); } } } @@ -3076,7 +3107,7 @@ public void start() { public void doReput() { if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) { LOGGER.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.", - this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); + this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset(); } for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) { @@ -3135,6 +3166,9 @@ public void doReput() { result.release(); } } + + // only for rocksdb mode + finishCommitLogDispatch(); } /** @@ -3177,8 +3211,8 @@ public void shutdown() { if (this.isCommitLogAvailable()) { LOGGER.warn("shutdown concurrentReputMessageService, but CommitLog have not finish to be dispatched, CommitLog max" + - " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(), - this.reputFromOffset); + " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(), + this.reputFromOffset); } this.mainBatchDispatchRequestService.shutdown(); diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java index 0bc70642fe9..9a0824829e1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java @@ -285,7 +285,7 @@ public long howMuchFallBehind() { if (this.mappedFiles.isEmpty()) return 0; - long committed = this.flushedWhere; + long committed = this.getFlushedWhere(); if (committed != 0) { MappedFile mappedFile = this.getLastMappedFile(0, false); if (mappedFile != null) { @@ -377,8 +377,19 @@ public MappedFile getLastMappedFile(final long startOffset) { } public MappedFile getLastMappedFile() { - MappedFile[] mappedFiles = this.mappedFiles.toArray(new MappedFile[0]); - return mappedFiles.length == 0 ? null : mappedFiles[mappedFiles.length - 1]; + MappedFile mappedFileLast = null; + while (!this.mappedFiles.isEmpty()) { + try { + mappedFileLast = this.mappedFiles.get(this.mappedFiles.size() - 1); + break; + } catch (IndexOutOfBoundsException e) { + //continue; + } catch (Exception e) { + log.error("getLastMappedFile has exception.", e); + break; + } + } + return mappedFileLast; } public boolean resetOffset(long offset) { @@ -442,11 +453,11 @@ public long getMaxWrotePosition() { } public long remainHowManyDataToCommit() { - return getMaxWrotePosition() - committedWhere; + return getMaxWrotePosition() - getCommittedWhere(); } public long remainHowManyDataToFlush() { - return getMaxOffset() - flushedWhere; + return getMaxOffset() - this.getFlushedWhere(); } public void deleteLastMappedFile() { @@ -616,15 +627,15 @@ public int deleteExpiredFileByOffsetForTimerLog(long offset, int checkOffset, in public boolean flush(final int flushLeastPages) { boolean result = true; - MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0); + MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0); if (mappedFile != null) { long tmpTimeStamp = mappedFile.getStoreTimestamp(); int offset = mappedFile.flush(flushLeastPages); long where = mappedFile.getFileFromOffset() + offset; - result = where == this.flushedWhere; - this.flushedWhere = where; + result = where == this.getFlushedWhere(); + this.setFlushedWhere(where); if (0 == flushLeastPages) { - this.storeTimestamp = tmpTimeStamp; + this.setStoreTimestamp(tmpTimeStamp); } } @@ -633,12 +644,12 @@ public boolean flush(final int flushLeastPages) { public synchronized boolean commit(final int commitLeastPages) { boolean result = true; - MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); + MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0); if (mappedFile != null) { int offset = mappedFile.commit(commitLeastPages); long where = mappedFile.getFileFromOffset() + offset; - result = where == this.committedWhere; - this.committedWhere = where; + result = where == this.getCommittedWhere(); + this.setCommittedWhere(where); } return result; @@ -763,7 +774,7 @@ public void destroy() { mf.destroy(1000 * 3); } this.mappedFiles.clear(); - this.flushedWhere = 0; + this.setFlushedWhere(0); // delete parent directory File file = new File(storePath); @@ -848,6 +859,10 @@ public long getStoreTimestamp() { return storeTimestamp; } + public void setStoreTimestamp(long storeTimestamp) { + this.storeTimestamp = storeTimestamp; + } + public List getMappedFiles() { return mappedFiles; } diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index 989cbbe3162..814c6d1bfef 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -16,10 +16,6 @@ */ package org.apache.rocketmq.store; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.ViewBuilder; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.LinkedList; @@ -40,10 +36,15 @@ import org.apache.rocketmq.store.hook.SendMessageBackHook; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; -import org.apache.rocketmq.store.queue.ConsumeQueueStore; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.PerfCounter; +import org.rocksdb.RocksDBException; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.ViewBuilder; /** * This class defines contracting interfaces to implement, allowing third-party vendor to use customized message store. @@ -545,7 +546,7 @@ CompletableFuture queryMessageAsync(final String topic, fina void setConfirmOffset(long phyOffset); /** - * Check if the operation system page cache is busy or not. + * Check if the operating system page cache is busy or not. * * @return true if the OS page cache is busy; false otherwise. */ @@ -620,9 +621,18 @@ CompletableFuture queryMessageAsync(final String topic, fina * @param commitLogFile commit log file * @param isRecover is from recover process * @param isFileEnd if the dispatch request represents 'file end' + * @throws RocksDBException only in rocksdb mode */ void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, - boolean isRecover, boolean isFileEnd); + boolean isRecover, boolean isFileEnd) throws RocksDBException; + + /** + * Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests) + * It will be triggered in two cases: + * @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput + * @see CommitLog#recoverAbnormally + */ + void finishCommitLogDispatch(); /** * Get the message store config @@ -691,13 +701,9 @@ void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, Ma * Truncate dirty logic files * * @param phyOffset physical offset + * @throws RocksDBException only in rocksdb mode */ - void truncateDirtyLogicFiles(long phyOffset); - - /** - * Destroy logics files - */ - void destroyLogics(); + void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException; /** * Unlock mappedFile @@ -718,7 +724,7 @@ void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, Ma * * @return the queue store */ - ConsumeQueueStore getQueueStore(); + ConsumeQueueStoreInterface getQueueStore(); /** * If 'sync disk flush' is configured in this message store @@ -739,8 +745,9 @@ void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, Ma * yourself. * * @param msg message + * @throws RocksDBException */ - void assignOffset(MessageExtBrokerInner msg); + void assignOffset(MessageExtBrokerInner msg) throws RocksDBException; /** * Increase queue offset in memory table. If there is a race condition, you need to lock/unlock this method @@ -835,14 +842,15 @@ void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, Ma * * @param offsetToTruncate offset to truncate * @return true if truncate succeed, false otherwise + * @throws RocksDBException only in rocksdb mode */ - boolean truncateFiles(long offsetToTruncate); + boolean truncateFiles(long offsetToTruncate) throws RocksDBException; /** - * Check if the offset is align with one message. + * Check if the offset is aligned with one message. * * @param offset offset to check - * @return true if align, false otherwise + * @return true if aligned, false otherwise */ boolean isOffsetAligned(long offset); @@ -971,4 +979,14 @@ DispatchRequest checkMessageAndReturnSize(final ByteBuffer byteBuffer, final boo * @param attributesBuilderSupplier metrics attributes builder */ void initMetrics(Meter meter, Supplier attributesBuilderSupplier); + + /** + * Recover topic queue table + */ + void recoverTopicQueueTable(); + + /** + * notify message arrive if necessary + */ + void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest); } diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java index 8f5af94380c..8ff050dfe3b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.store; - import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -113,8 +112,7 @@ public void destroy() { mf.destroy(1000 * 3); } this.mappedFiles.clear(); - this.flushedWhere = 0; - + this.setFlushedWhere(0); Set storePathSet = getPaths(); storePathSet.addAll(getReadonlyPaths()); diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java new file mode 100644 index 00000000000..87ccb5474b6 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; +import org.apache.rocketmq.store.queue.RocksDBConsumeQueue; +import org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.rocksdb.RocksDBException; + +public class RocksDBMessageStore extends DefaultMessageStore { + + public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, + final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws + IOException { + super(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, topicConfigTable); + notifyMessageArriveInBatch = true; + } + + @Override + public ConsumeQueueStoreInterface createConsumeQueueStore() { + return new RocksDBConsumeQueueStore(this); + } + + @Override + public CleanConsumeQueueService createCleanConsumeQueueService() { + return new RocksDBCleanConsumeQueueService(); + } + + @Override + public FlushConsumeQueueService createFlushConsumeQueueService() { + return new RocksDBFlushConsumeQueueService(); + } + + @Override + public CorrectLogicOffsetService createCorrectLogicOffsetService() { + return new RocksDBCorrectLogicOffsetService(); + } + + /** + * Try to set topicQueueTable = new HashMap<>(), otherwise it will cause bug when broker role changes. + * And unlike method in DefaultMessageStore, we don't need to really recover topic queue table advance, + * because we can recover topic queue table from rocksdb when we need to use it. + * @see RocksDBConsumeQueue#assignQueueOffset + */ + @Override + public void recoverTopicQueueTable() { + this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>()); + } + + @Override + public void finishCommitLogDispatch() { + try { + putMessagePositionInfo(null); + } catch (RocksDBException e) { + ERROR_LOG.info("try to finish commitlog dispatch error.", e); + } + } + + @Override + public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) { + return findConsumeQueue(topic, queueId); + } + + class RocksDBCleanConsumeQueueService extends CleanConsumeQueueService { + private final double diskSpaceWarningLevelRatio = + Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceWarningLevelRatio", "0.90")); + + private final double diskSpaceCleanForciblyRatio = + Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "0.85")); + + @Override + protected void deleteExpiredFiles() { + + long minOffset = RocksDBMessageStore.this.commitLog.getMinOffset(); + if (minOffset > this.lastPhysicalMinOffset) { + this.lastPhysicalMinOffset = minOffset; + + boolean spaceFull = isSpaceToDelete(); + boolean timeUp = cleanCommitLogService.isTimeToDelete(); + if (spaceFull || timeUp) { + RocksDBMessageStore.this.consumeQueueStore.cleanExpired(minOffset); + } + + RocksDBMessageStore.this.indexService.deleteExpiredFile(minOffset); + } + } + + private boolean isSpaceToDelete() { + double ratio = RocksDBMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0; + + String storePathLogics = StorePathConfigHelper + .getStorePathConsumeQueue(RocksDBMessageStore.this.getMessageStoreConfig().getStorePathRootDir()); + double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics); + if (logicsRatio > diskSpaceWarningLevelRatio) { + boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskFull(); + if (diskOk) { + RocksDBMessageStore.LOGGER.error("logics disk maybe full soon " + logicsRatio + ", so mark disk full"); + } + } else if (logicsRatio > diskSpaceCleanForciblyRatio) { + } else { + boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskOK(); + if (!diskOk) { + RocksDBMessageStore.LOGGER.info("logics disk space OK " + logicsRatio + ", so mark disk ok"); + } + } + + if (logicsRatio < 0 || logicsRatio > ratio) { + RocksDBMessageStore.LOGGER.info("logics disk maybe full soon, so reclaim space, " + logicsRatio); + return true; + } + + return false; + } + } + + class RocksDBFlushConsumeQueueService extends FlushConsumeQueueService { + /** + * There is no need to flush consume queue, + * we put all consume queues in RocksDBConsumeQueueStore, + * it depends on rocksdb to flush consume queue to disk(sorted string table), + * we even don't flush WAL of consume store, since we think it can recover consume queue from commitlog. + */ + @Override + public void run() { + + } + } + + class RocksDBCorrectLogicOffsetService extends CorrectLogicOffsetService { + /** + * There is no need to correct min offset of consume queue, we already fix this problem. + * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset + */ + public void run() { + + } + } + + @Override + public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) { + // todo + return 0; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java index 2ae6879aadb..91fcb155a65 100644 --- a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java +++ b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java @@ -30,6 +30,8 @@ public class RunningFlags { private static final int FENCED_BIT = 1 << 5; + private static final int LOGIC_DISK_FULL_BIT = 1 << 5; + private volatile int flagBits = 0; public RunningFlags() { @@ -63,6 +65,10 @@ public boolean getAndMakeNotReadable() { return result; } + public void clearLogicsQueueError() { + this.flagBits &= ~WRITE_LOGICS_QUEUE_ERROR_BIT; + } + public boolean getAndMakeWriteable() { boolean result = this.isWriteable(); if (!result) { @@ -72,7 +78,7 @@ public boolean getAndMakeWriteable() { } public boolean isWriteable() { - if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT)) == 0) { + if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT | LOGIC_DISK_FULL_BIT)) == 0) { return true; } @@ -81,7 +87,7 @@ public boolean isWriteable() { //for consume queue, just ignore the DISK_FULL_BIT public boolean isCQWriteable() { - if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT)) == 0) { + if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT | LOGIC_DISK_FULL_BIT)) == 0) { return true; } @@ -139,4 +145,16 @@ public boolean getAndMakeDiskOK() { this.flagBits &= ~DISK_FULL_BIT; return result; } + + public boolean getAndMakeLogicDiskFull() { + boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT); + this.flagBits |= LOGIC_DISK_FULL_BIT; + return result; + } + + public boolean getAndMakeLogicDiskOK() { + boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT); + this.flagBits &= ~LOGIC_DISK_FULL_BIT; + return result; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java index a78eeed230a..5a131b5c35c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java +++ b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java @@ -34,6 +34,14 @@ public TopicQueueLock() { } } + public TopicQueueLock(int size) { + this.size = size; + this.lockList = new ArrayList<>(size); + for (int i = 0; i < this.size; i++) { + this.lockList.add(new ReentrantLock()); + } + } + public void lock(String topicQueueKey) { Lock lock = this.lockList.get((topicQueueKey.hashCode() & 0x7fffffff) % this.size); lock.lock(); diff --git a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java index 8c1a5338b98..0d42ee69e68 100644 --- a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java +++ b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java @@ -33,13 +33,11 @@ public class TransientStorePool { private final int poolSize; private final int fileSize; private final Deque availableBuffers; - private final DefaultMessageStore messageStore; private volatile boolean isRealCommit = true; - public TransientStorePool(final DefaultMessageStore messageStore) { - this.messageStore = messageStore; - this.poolSize = messageStore.getMessageStoreConfig().getTransientStorePoolSize(); - this.fileSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); + public TransientStorePool(final int poolSize, final int fileSize) { + this.poolSize = poolSize; + this.fileSize = fileSize; this.availableBuffers = new ConcurrentLinkedDeque<>(); } @@ -81,10 +79,7 @@ public ByteBuffer borrowBuffer() { } public int availableBufferNums() { - if (messageStore.isTransientStorePoolEnable()) { - return availableBuffers.size(); - } - return Integer.MAX_VALUE; + return availableBuffers.size(); } public boolean isRealCommit() { diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index efb728ac04a..028facbdc6d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -397,10 +397,14 @@ public class MessageStoreConfig { private int batchDispatchRequestThreadPoolNums = 16; // rocksdb mode + private long cleanRocksDBDirtyCQIntervalMin = 60; + private long statRocksDBCQIntervalSec = 10; + private long memTableFlushIntervalMs = 60 * 60 * 1000L; private boolean realTimePersistRocksDBConfig = true; - private long memTableFlushInterval = 60 * 60 * 1000L; private boolean enableRocksDBLog = false; + private int topicQueueLockNum = 32; + public boolean isDebugLockEnable() { return debugLockEnable; } @@ -497,6 +501,10 @@ public void setMappedFileSizeCommitLog(int mappedFileSizeCommitLog) { this.mappedFileSizeCommitLog = mappedFileSizeCommitLog; } + public boolean isEnableRocksDBStore() { + return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.storeType); + } + public String getStoreType() { return storeType; } @@ -506,7 +514,6 @@ public void setStoreType(String storeType) { } public int getMappedFileSizeConsumeQueue() { - int factor = (int) Math.ceil(this.mappedFileSizeConsumeQueue / (ConsumeQueue.CQ_STORE_UNIT_SIZE * 1.0)); return (int) (factor * ConsumeQueue.CQ_STORE_UNIT_SIZE); } @@ -1736,12 +1743,28 @@ public void setRealTimePersistRocksDBConfig(boolean realTimePersistRocksDBConfig this.realTimePersistRocksDBConfig = realTimePersistRocksDBConfig; } - public long getMemTableFlushInterval() { - return memTableFlushInterval; + public long getStatRocksDBCQIntervalSec() { + return statRocksDBCQIntervalSec; + } + + public void setStatRocksDBCQIntervalSec(long statRocksDBCQIntervalSec) { + this.statRocksDBCQIntervalSec = statRocksDBCQIntervalSec; + } + + public long getCleanRocksDBDirtyCQIntervalMin() { + return cleanRocksDBDirtyCQIntervalMin; } - public void setMemTableFlushInterval(long memTableFlushInterval) { - this.memTableFlushInterval = memTableFlushInterval; + public void setCleanRocksDBDirtyCQIntervalMin(long cleanRocksDBDirtyCQIntervalMin) { + this.cleanRocksDBDirtyCQIntervalMin = cleanRocksDBDirtyCQIntervalMin; + } + + public long getMemTableFlushIntervalMs() { + return memTableFlushIntervalMs; + } + + public void setMemTableFlushIntervalMs(long memTableFlushIntervalMs) { + this.memTableFlushIntervalMs = memTableFlushIntervalMs; } public boolean isEnableRocksDBLog() { @@ -1751,4 +1774,12 @@ public boolean isEnableRocksDBLog() { public void setEnableRocksDBLog(boolean enableRocksDBLog) { this.enableRocksDBLog = enableRocksDBLog; } + + public int getTopicQueueLockNum() { + return topicQueueLockNum; + } + + public void setTopicQueueLockNum(int topicQueueLockNum) { + this.topicQueueLockNum = topicQueueLockNum; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index ec5e86d704d..70371d83b83 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -16,26 +16,13 @@ */ package org.apache.rocketmq.store.dledger; -import io.openmessaging.storage.dledger.AppendFuture; -import io.openmessaging.storage.dledger.BatchAppendFuture; -import io.openmessaging.storage.dledger.DLedgerConfig; -import io.openmessaging.storage.dledger.DLedgerServer; -import io.openmessaging.storage.dledger.entry.DLedgerEntry; -import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; -import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; -import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; -import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; -import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; -import io.openmessaging.storage.dledger.store.file.MmapFile; -import io.openmessaging.storage.dledger.store.file.MmapFileList; -import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; -import io.openmessaging.storage.dledger.utils.DLedgerUtils; import java.net.Inet6Address; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; + import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBatch; @@ -54,6 +41,22 @@ import org.apache.rocketmq.store.StoreStatsService; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.logfile.MappedFile; +import org.rocksdb.RocksDBException; + +import io.openmessaging.storage.dledger.AppendFuture; +import io.openmessaging.storage.dledger.BatchAppendFuture; +import io.openmessaging.storage.dledger.DLedgerConfig; +import io.openmessaging.storage.dledger.DLedgerServer; +import io.openmessaging.storage.dledger.entry.DLedgerEntry; +import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; +import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; +import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; +import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; +import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; +import io.openmessaging.storage.dledger.store.file.MmapFile; +import io.openmessaging.storage.dledger.store.file.MmapFileList; +import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; +import io.openmessaging.storage.dledger.utils.DLedgerUtils; /** * Store all metadata downtime for recovery, data protection reliability @@ -162,7 +165,12 @@ public long getMinOffset() { if (!mappedFileQueue.getMappedFiles().isEmpty()) { return mappedFileQueue.getMinOffset(); } - return dLedgerFileList.getMinOffset(); + for (MmapFile file : dLedgerFileList.getMappedFiles()) { + if (file.isAvailable()) { + return file.getFileFromOffset() + file.getStartPosition(); + } + } + return 0; } @Override @@ -264,7 +272,7 @@ public SelectMappedBufferResult getData(final long offset, final boolean returnF return null; } - + @Override public boolean getData(final long offset, final int size, final ByteBuffer byteBuffer) { if (offset < dividedCommitlogOffset) { @@ -282,7 +290,7 @@ public boolean getData(final long offset, final int size, final ByteBuffer byteB return false; } - private void recover(long maxPhyOffsetOfConsumeQueue) { + private void recover(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { dLedgerFileStore.load(); if (dLedgerFileList.getMappedFiles().size() > 0) { dLedgerFileStore.recover(); @@ -336,12 +344,12 @@ private void recover(long maxPhyOffsetOfConsumeQueue) { } @Override - public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { + public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { recover(maxPhyOffsetOfConsumeQueue); } @Override - public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { + public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { recover(maxPhyOffsetOfConsumeQueue); } @@ -464,9 +472,6 @@ public CompletableFuture asyncPutMessage(MessageExtBrokerInner String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock); - } catch (Exception e) { - log.error("Put message error", e); - return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); } finally { beginTimeInDledgerLock = 0; putMessageLock.unlock(); @@ -477,6 +482,9 @@ public CompletableFuture asyncPutMessage(MessageExtBrokerInner } defaultMessageStore.increaseOffset(msg, getMessageNum(msg)); + } catch (Exception e) { + log.error("Put message error", e); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); } finally { topicQueueLock.unlock(topicQueueKey); } @@ -606,9 +614,6 @@ public CompletableFuture asyncPutMessages(MessageExtBatch mess appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen, msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock); appendResult.setMsgNum(msgNum); - } catch (Exception e) { - log.error("Put message error", e); - return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); } finally { beginTimeInDledgerLock = 0; putMessageLock.unlock(); @@ -621,7 +626,10 @@ public CompletableFuture asyncPutMessages(MessageExtBatch mess defaultMessageStore.increaseOffset(messageExtBatch, (short) batchNum); - } finally { + } catch (Exception e) { + log.error("Put message error", e); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { topicQueueLock.unlock(encodeResult.queueOffsetKey); } diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java index 467da603d47..aaea7d6900a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.store.CommitLog; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.rocksdb.RocksDBException; public interface HAService { @@ -53,7 +54,7 @@ public interface HAService { * * @param masterEpoch the new masterEpoch */ - default boolean changeToMaster(int masterEpoch) { + default boolean changeToMaster(int masterEpoch) throws RocksDBException { return false; } diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java index 936db0c4c6e..176c25a96f6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java @@ -432,7 +432,7 @@ public void run() { /** * Compare the master and slave's epoch file, find consistent point, do truncate. */ - private boolean doTruncate(List masterEpochEntries, long masterEndOffset) throws IOException { + private boolean doTruncate(List masterEpochEntries, long masterEndOffset) throws Exception { if (this.epochCache.getEntrySize() == 0) { // If epochMap is empty, means the broker is a new replicas LOGGER.info("Slave local epochCache is empty, skip truncate log"); diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java index 6dc734e0c97..64dad9aef21 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java @@ -17,10 +17,26 @@ package org.apache.rocketmq.store.ha.autoswitch; - +import java.io.IOException; +import java.nio.channels.SocketChannel; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Consumer; +import java.util.stream.Collectors; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.protocol.EpochEntry; @@ -35,30 +51,14 @@ import org.apache.rocketmq.store.ha.HAClient; import org.apache.rocketmq.store.ha.HAConnection; import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService; - -import java.io.IOException; -import java.nio.channels.SocketChannel; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Consumer; -import java.util.stream.Collectors; +import org.rocksdb.RocksDBException; /** * SwitchAble ha service, support switch role to master or slave. */ public class AutoSwitchHAService extends DefaultHAService { private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); - private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_")); + private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_")); private final ConcurrentHashMap connectionCaughtUpTimeTable = new ConcurrentHashMap<>(); private final List>> syncStateSetChangedListeners = new ArrayList<>(); private final Set syncStateSet = new HashSet<>(); @@ -73,7 +73,7 @@ public class AutoSwitchHAService extends DefaultHAService { private EpochFileCache epochCache; private AutoSwitchHAClient haClient; - private Long brokerControllerId = null; + private Long localBrokerId = null; public AutoSwitchHAService() { } @@ -112,7 +112,7 @@ public void removeConnection(HAConnection conn) { } @Override - public boolean changeToMaster(int masterEpoch) { + public boolean changeToMaster(int masterEpoch) throws RocksDBException { final int lastEpoch = this.epochCache.lastEpoch(); if (masterEpoch < lastEpoch) { LOGGER.warn("newMasterEpoch {} < lastEpoch {}, fail to change to master", masterEpoch, lastEpoch); @@ -287,9 +287,11 @@ public Set maybeShrinkSyncStateSet() { // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable, // it means that the broker has not connected. - for (Long slaveBrokerId : newSyncStateSet) { - if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { - newSyncStateSet.remove(slaveBrokerId); + Iterator iterator = newSyncStateSet.iterator(); + while (iterator.hasNext()) { + Long slaveBrokerId = iterator.next(); + if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { + iterator.remove(); isSyncStateSetChanged = true; } } @@ -314,7 +316,7 @@ public void maybeExpandInSyncStateSet(final Long slaveBrokerId, final long slave final EpochEntry currentLeaderEpoch = this.epochCache.lastEntry(); if (slaveMaxOffset >= currentLeaderEpoch.getStartOffset()) { LOGGER.info("The slave {} has caught up, slaveMaxOffset: {}, confirmOffset: {}, epoch: {}, leader epoch startOffset: {}.", - slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset()); + slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset()); currentSyncStateSet.add(slaveBrokerId); markSynchronizingSyncStateSet(currentSyncStateSet); // Notify the upper layer that syncStateSet changed. @@ -419,7 +421,7 @@ public long computeConfirmOffset() { // To avoid the syncStateSet is not consistent with connectionList. // Fix issue: https://github.com/apache/rocketmq/issues/6662 for (Long syncId : currentSyncStateSet) { - if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) { + if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) { LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId); // Without check and re-compute, return the confirmOffset's value directly. return this.defaultMessageStore.getConfirmOffsetDirectly(); @@ -490,7 +492,7 @@ public void truncateEpochFileSuffix(final long offset) { /** * Try to truncate incomplete msg transferred from master. */ - public long truncateInvalidMsg() { + public long truncateInvalidMsg() throws RocksDBException { long dispatchBehind = this.defaultMessageStore.dispatchBehindBytes(); if (dispatchBehind <= 0) { LOGGER.info("Dispatch complete, skip truncate"); @@ -545,12 +547,12 @@ public List getEpochEntries() { return this.epochCache.getAllEntries(); } - public Long getBrokerControllerId() { - return brokerControllerId; + public Long getLocalBrokerId() { + return localBrokerId; } - public void setBrokerControllerId(Long brokerControllerId) { - this.brokerControllerId = brokerControllerId; + public void setLocalBrokerId(Long localBrokerId) { + this.localBrokerId = localBrokerId; } class AutoSwitchAcceptSocketService extends AcceptSocketService { diff --git a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java index b37c907267c..639084fa2d8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java @@ -16,17 +16,25 @@ */ package org.apache.rocketmq.store.kv; -import java.util.Random; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.attribute.CleanupPolicy; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.CleanupPolicyUtils; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; @@ -35,15 +43,6 @@ import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.MessageStoreConfig; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - public class CompactionStore { public static final String COMPACTION_DIR = "compaction"; @@ -76,7 +75,7 @@ public CompactionStore(DefaultMessageStore defaultMessageStore) { this.positionMgr = new CompactionPositionMgr(compactionPath); this.compactionThreadNum = Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, config.getCompactionThreadNum())); - this.compactionSchedule = Executors.newScheduledThreadPool(this.compactionThreadNum, + this.compactionSchedule = ThreadUtils.newScheduledThreadPool(this.compactionThreadNum, new ThreadFactoryImpl("compactionSchedule_")); this.offsetMapSize = config.getMaxOffsetMapSize() / compactionThreadNum; diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index ab9fc6da730..2f2ce981257 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -17,10 +17,6 @@ package org.apache.rocketmq.store.plugin; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.ViewBuilder; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.LinkedList; @@ -55,10 +51,16 @@ import org.apache.rocketmq.store.hook.SendMessageBackHook; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; -import org.apache.rocketmq.store.queue.ConsumeQueueStore; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.PerfCounter; +import org.rocksdb.RocksDBException; + +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.ViewBuilder; public abstract class AbstractPluginMessageStore implements MessageStore { protected MessageStore next = null; @@ -457,7 +459,7 @@ public HAService getHaService() { } @Override - public boolean truncateFiles(long offsetToTruncate) { + public boolean truncateFiles(long offsetToTruncate) throws RocksDBException { return next.truncateFiles(offsetToTruncate); } @@ -511,7 +513,7 @@ public void onCommitLogAppend(MessageExtBrokerInner msg, AppendMessageResult res @Override public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, - boolean isRecover, boolean isFileEnd) { + boolean isRecover, boolean isFileEnd) throws RocksDBException { next.onCommitLogDispatch(dispatchRequest, doDispatch, commitLogFile, isRecover, isFileEnd); } @@ -551,15 +553,10 @@ public AllocateMappedFileService getAllocateMappedFileService() { } @Override - public void truncateDirtyLogicFiles(long phyOffset) { + public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException { next.truncateDirtyLogicFiles(phyOffset); } - @Override - public void destroyLogics() { - next.destroyLogics(); - } - @Override public void unlockMappedFile(MappedFile unlockMappedFile) { next.unlockMappedFile(unlockMappedFile); @@ -571,7 +568,7 @@ public PerfCounter.Ticks getPerfCounter() { } @Override - public ConsumeQueueStore getQueueStore() { + public ConsumeQueueStoreInterface getQueueStore() { return next.getQueueStore(); } @@ -586,7 +583,7 @@ public boolean isSyncMaster() { } @Override - public void assignOffset(MessageExtBrokerInner msg) { + public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException { next.assignOffset(msg); } @@ -649,4 +646,19 @@ public List> getMetricsView() { public void initMetrics(Meter meter, Supplier attributesBuilderSupplier) { next.initMetrics(meter, attributesBuilderSupplier); } + + @Override + public void finishCommitLogDispatch() { + next.finishCommitLogDispatch(); + } + + @Override + public void recoverTopicQueueTable() { + next.recoverTopicQueueTable(); + } + + @Override + public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { + next.notifyMessageArriveIfNecessary(dispatchRequest); + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java new file mode 100644 index 00000000000..30054fa509a --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.rocksdb.RocksDBException; + +public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface { + protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + + protected final DefaultMessageStore messageStore; + protected final MessageStoreConfig messageStoreConfig; + protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator(); + protected final ConcurrentMap> consumeQueueTable; + + public AbstractConsumeQueueStore(DefaultMessageStore messageStore) { + this.messageStore = messageStore; + this.messageStoreConfig = messageStore.getMessageStoreConfig(); + this.consumeQueueTable = new ConcurrentHashMap<>(32); + } + + @Override + public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) { + consumeQueue.putMessagePositionInfoWrapper(request); + } + + @Override + public Long getMaxOffset(String topic, int queueId) { + return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); + } + + @Override + public void setTopicQueueTable(ConcurrentMap topicQueueTable) { + this.queueOffsetOperator.setTopicQueueTable(topicQueueTable); + this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable); + } + + @Override + public ConcurrentMap getTopicQueueTable() { + return this.queueOffsetOperator.getTopicQueueTable(); + } + + @Override + public void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException { + ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); + consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg); + } + + @Override + public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { + ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); + consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum); + } + + @Override + public void removeTopicQueueTable(String topic, Integer queueId) { + this.queueOffsetOperator.remove(topic, queueId); + } + + @Override + public ConcurrentMap> getConsumeQueueTable() { + return this.consumeQueueTable; + } + + @Override + public ConcurrentMap findConsumeQueueMap(String topic) { + return this.consumeQueueTable.get(topic); + } + + @Override + public long getStoreTime(CqUnit cqUnit) { + if (cqUnit != null) { + try { + final long phyOffset = cqUnit.getPos(); + final int size = cqUnit.getSize(); + long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); + return storeTime; + } catch (Exception e) { + } + } + return -1; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java index 387c233bf56..7108c835c8e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java @@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.function.Function; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.attribute.CQType; import org.apache.rocketmq.common.constant.LoggerName; @@ -311,6 +312,11 @@ public ReferredIterator iterateFrom(long startOffset) { return new BatchConsumeQueueIterator(sbr); } + @Override + public ReferredIterator iterateFrom(long startIndex, int count) { + return iterateFrom(startIndex); + } + @Override public CqUnit get(long offset) { ReferredIterator it = iterateFrom(offset); @@ -320,6 +326,20 @@ public CqUnit get(long offset) { return it.nextAndRelease(); } + @Override + public Pair getCqUnitAndStoreTime(long index) { + CqUnit cqUnit = get(index); + Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); + return new Pair<>(cqUnit, messageStoreTime); + } + + @Override + public Pair getEarliestUnitAndStoreTime() { + CqUnit cqUnit = getEarliestUnit(); + Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); + return new Pair<>(cqUnit, messageStoreTime); + } + @Override public CqUnit getEarliestUnit() { return get(minOffsetInQueue); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java index 55d08082925..c65f2a68b05 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java @@ -18,10 +18,12 @@ package org.apache.rocketmq.store.queue; import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.attribute.CQType; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.MessageFilter; +import org.rocksdb.RocksDBException; public interface ConsumeQueueInterface extends FileQueueLifeCycle { /** @@ -44,6 +46,16 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { */ ReferredIterator iterateFrom(long startIndex); + /** + * Get the units from the start offset. + * + * @param startIndex start index + * @param count the unit counts will be iterated + * @return the unit iterateFrom + * @throws RocksDBException only in rocksdb mode + */ + ReferredIterator iterateFrom(long startIndex, int count) throws RocksDBException; + /** * Get cq unit at specified index * @param index index @@ -51,6 +63,18 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { */ CqUnit get(long index); + /** + * Get earliest cq unit + * @return the cq unit and message storeTime at index + */ + Pair getCqUnitAndStoreTime(long index); + + /** + * Get earliest cq unit + * @return earliest cq unit and message storeTime + */ + Pair getEarliestUnitAndStoreTime(); + /** * Get earliest cq unit * @return earliest cq unit @@ -153,8 +177,9 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { * Assign queue offset. * @param queueOffsetAssigner the delegated queue offset assigner * @param msg message itself + * @throws RocksDBException only in rocksdb mode */ - void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg); + void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg) throws RocksDBException; /** diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java index 8d38503b371..616511b67fb 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java @@ -16,64 +16,128 @@ */ package org.apache.rocketmq.store.queue; +import java.io.File; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.attribute.CQType; -import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.QueueTypeUtils; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.store.CommitLog; import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; -import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.SelectMappedBufferResult; -import org.apache.rocketmq.store.config.MessageStoreConfig; - -import java.io.File; -import java.util.Iterator; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import static java.lang.String.format; import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue; import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue; -public class ConsumeQueueStore { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); +public class ConsumeQueueStore extends AbstractConsumeQueueStore { - protected final DefaultMessageStore messageStore; - protected final MessageStoreConfig messageStoreConfig; - protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator(); - protected final ConcurrentMap> consumeQueueTable; + public ConsumeQueueStore(DefaultMessageStore messageStore) { + super(messageStore); + } - public ConsumeQueueStore(DefaultMessageStore messageStore, MessageStoreConfig messageStoreConfig) { - this.messageStore = messageStore; - this.messageStoreConfig = messageStoreConfig; - this.consumeQueueTable = new ConcurrentHashMap<>(32); + @Override + public void start() { + log.info("Default ConsumeQueueStore start!"); } - private FileQueueLifeCycle getLifeCycle(String topic, int queueId) { - return findOrCreateConsumeQueue(topic, queueId); + @Override + public boolean load() { + boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ); + boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ); + return cqLoadResult && bcqLoadResult; + } + + @Override + public boolean loadAfterDestroy() { + return true; + } + + @Override + public void recover() { + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueueInterface logic : maps.values()) { + this.recover(logic); + } + } } + @Override + public boolean recoverConcurrently() { + int count = 0; + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + count += maps.values().size(); + } + final CountDownLatch countDownLatch = new CountDownLatch(count); + BlockingQueue recoverQueue = new LinkedBlockingQueue<>(); + final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_"); + List> result = new ArrayList<>(count); + try { + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (final ConsumeQueueInterface logic : maps.values()) { + FutureTask futureTask = new FutureTask<>(() -> { + boolean ret = true; + try { + logic.recover(); + } catch (Throwable e) { + ret = false; + log.error("Exception occurs while recover consume queue concurrently, " + + "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e); + } finally { + countDownLatch.countDown(); + } + return ret; + }); + + result.add(futureTask); + executor.submit(futureTask); + } + } + countDownLatch.await(); + for (FutureTask task : result) { + if (task != null && task.isDone()) { + if (!task.get()) { + return false; + } + } + } + } catch (Exception e) { + log.error("Exception occurs while recover consume queue concurrently", e); + return false; + } finally { + executor.shutdown(); + } + return true; + } + + @Override + public boolean shutdown() { + return true; + } + + @Override public long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.rollNextFile(offset); @@ -83,32 +147,53 @@ public void correctMinOffset(ConsumeQueueInterface consumeQueue, long minCommitL consumeQueue.correctMinOffset(minCommitLogOffset); } - /** - * Apply the dispatched request and build the consume queue. This function should be idempotent. - * - * @param consumeQueue consume queue - * @param request dispatch request - */ - public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) { - consumeQueue.putMessagePositionInfoWrapper(request); - } - + @Override public void putMessagePositionInfoWrapper(DispatchRequest dispatchRequest) { ConsumeQueueInterface cq = this.findOrCreateConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId()); this.putMessagePositionInfoWrapper(cq, dispatchRequest); } + @Override + public List rangeQuery(String topic, int queueId, long startIndex, int num) { + return null; + } + + @Override + public ByteBuffer get(String topic, int queueId, long startIndex) { + return null; + } + + @Override + public long getMaxOffsetInQueue(String topic, int queueId) { + ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMaxOffsetInQueue(); + } + return 0; + } + + @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) { + ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); + if (logic != null) { + long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType); + // Make sure the result offset is in valid range. + resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue()); + resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue()); + return resultOffset; + } + return 0; + } + + private FileQueueLifeCycle getLifeCycle(String topic, int queueId) { + return findOrCreateConsumeQueue(topic, queueId); + } + public boolean load(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.load(); } - public boolean load() { - boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ); - boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ); - return cqLoadResult && bcqLoadResult; - } - private boolean loadConsumeQueues(String storePath, CQType cqType) { File dirLogic = new File(storePath); File[] fileTopicList = dirLogic.listFiles(); @@ -175,7 +260,7 @@ private void queueTypeShouldBe(String topic, CQType cqTypeExpected) { } private ExecutorService buildExecutorService(BlockingQueue blockingQueue, String threadNamePrefix) { - return new ThreadPoolExecutor( + return ThreadUtils.newThreadPoolExecutor( this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(), this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(), 1000 * 60, @@ -189,62 +274,17 @@ public void recover(ConsumeQueueInterface consumeQueue) { fileQueueLifeCycle.recover(); } - public void recover() { - for (ConcurrentMap maps : this.consumeQueueTable.values()) { - for (ConsumeQueueInterface logic : maps.values()) { - this.recover(logic); - } - } - } - - public boolean recoverConcurrently() { - int count = 0; - for (ConcurrentMap maps : this.consumeQueueTable.values()) { - count += maps.values().size(); - } - final CountDownLatch countDownLatch = new CountDownLatch(count); - BlockingQueue recoverQueue = new LinkedBlockingQueue<>(); - final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_"); - List> result = new ArrayList<>(count); - try { - for (ConcurrentMap maps : this.consumeQueueTable.values()) { - for (final ConsumeQueueInterface logic : maps.values()) { - FutureTask futureTask = new FutureTask<>(() -> { - boolean ret = true; - try { - logic.recover(); - } catch (Throwable e) { - ret = false; - log.error("Exception occurs while recover consume queue concurrently, " + - "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e); - } finally { - countDownLatch.countDown(); - } - return ret; - }); - - result.add(futureTask); - executor.submit(futureTask); - } - } - countDownLatch.await(); - for (FutureTask task : result) { - if (task != null && task.isDone()) { - if (!task.get()) { - return false; - } - } - } - } catch (Exception e) { - log.error("Exception occurs while recover consume queue concurrently", e); - return false; - } finally { - executor.shutdown(); + @Override + public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) { + ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMaxPhysicOffset(); } - return true; + return null; } - public long getMaxOffsetInConsumeQueue() { + @Override + public long getMaxPhyOffsetInConsumeQueue() { long maxPhysicOffset = -1L; for (ConcurrentMap maps : this.consumeQueueTable.values()) { for (ConsumeQueueInterface logic : maps.values()) { @@ -256,11 +296,22 @@ public long getMaxOffsetInConsumeQueue() { return maxPhysicOffset; } + @Override + public long getMinOffsetInQueue(String topic, int queueId) { + ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMinOffsetInQueue(); + } + + return -1; + } + public void checkSelf(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); fileQueueLifeCycle.checkSelf(); } + @Override public void checkSelf() { for (Map.Entry> topicEntry : this.consumeQueueTable.entrySet()) { for (Map.Entry cqEntry : topicEntry.getValue().entrySet()) { @@ -269,16 +320,19 @@ public void checkSelf() { } } + @Override public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.flush(flushLeastPages); } + @Override public void destroy(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); fileQueueLifeCycle.destroy(); } + @Override public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.deleteExpiredFile(minCommitLogPos); @@ -300,21 +354,20 @@ public void cleanSwappedMap(ConsumeQueueInterface consumeQueue, long forceCleanS fileQueueLifeCycle.cleanSwappedMap(forceCleanSwapIntervalMs); } + @Override public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.isFirstFileAvailable(); } + @Override public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); return fileQueueLifeCycle.isFirstFileExist(); } + @Override public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { - return doFindOrCreateConsumeQueue(topic, queueId); - } - - private ConsumeQueueInterface doFindOrCreateConsumeQueue(String topic, int queueId) { ConcurrentMap map = consumeQueueTable.get(topic); if (null == map) { ConcurrentMap newMap = new ConcurrentHashMap<>(128); @@ -361,46 +414,15 @@ private ConsumeQueueInterface doFindOrCreateConsumeQueue(String topic, int queue return logic; } - public Long getMaxOffset(String topic, int queueId) { - return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); - } - - public void setTopicQueueTable(ConcurrentMap topicQueueTable) { - this.queueOffsetOperator.setTopicQueueTable(topicQueueTable); - this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable); - } - - public ConcurrentMap getTopicQueueTable() { - return this.queueOffsetOperator.getTopicQueueTable(); - } - public void setBatchTopicQueueTable(ConcurrentMap batchTopicQueueTable) { this.queueOffsetOperator.setBatchTopicQueueTable(batchTopicQueueTable); } - public void assignQueueOffset(MessageExtBrokerInner msg) { - ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); - consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg); - } - - public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { - ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); - consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum); - } - public void updateQueueOffset(String topic, int queueId, long offset) { String topicQueueKey = topic + "-" + queueId; this.queueOffsetOperator.updateQueueOffset(topicQueueKey, offset); } - public void removeTopicQueueTable(String topic, Integer queueId) { - this.queueOffsetOperator.remove(topic, queueId); - } - - public ConcurrentMap> getConsumeQueueTable() { - return consumeQueueTable; - } - private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueueInterface consumeQueue) { ConcurrentMap map = this.consumeQueueTable.get(topic); if (null == map) { @@ -412,6 +434,7 @@ private void putConsumeQueue(final String topic, final int queueId, final Consum } } + @Override public void recoverOffsetTable(long minPhyOffset) { ConcurrentMap cqOffsetTable = new ConcurrentHashMap<>(1024); ConcurrentMap bcqOffsetTable = new ConcurrentHashMap<>(1024); @@ -431,7 +454,7 @@ public void recoverOffsetTable(long minPhyOffset) { } } - //Correct unSubmit consumeOffset + // Correct unSubmit consumeOffset if (messageStoreConfig.isDuplicationEnable()) { SelectMappedBufferResult lastBuffer = null; long startReadOffset = messageStore.getCommitLog().getConfirmOffset() == -1 ? 0 : messageStore.getCommitLog().getConfirmOffset(); @@ -476,6 +499,7 @@ public void recoverOffsetTable(long minPhyOffset) { this.setBatchTopicQueueTable(bcqOffsetTable); } + @Override public void destroy() { for (ConcurrentMap maps : this.consumeQueueTable.values()) { for (ConsumeQueueInterface logic : maps.values()) { @@ -484,8 +508,9 @@ public void destroy() { } } + @Override public void cleanExpired(long minCommitLogOffset) { - Iterator>> it = this.consumeQueueTable.entrySet().iterator(); + Iterator>> it = this.consumeQueueTable.entrySet().iterator(); while (it.hasNext()) { Map.Entry> next = it.next(); String topic = next.getKey(); @@ -526,14 +551,16 @@ public void cleanExpired(long minCommitLogOffset) { } } - public void truncateDirty(long phyOffset) { + @Override + public void truncateDirty(long offsetToTruncate) { for (ConcurrentMap maps : this.consumeQueueTable.values()) { for (ConsumeQueueInterface logic : maps.values()) { - this.truncateDirtyLogicFiles(logic, phyOffset); + this.truncateDirtyLogicFiles(logic, offsetToTruncate); } } } + @Override public long getTotalSize() { long totalSize = 0; for (ConcurrentMap maps : this.consumeQueueTable.values()) { diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java new file mode 100644 index 00000000000..268803dccad --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.store.DispatchRequest; +import org.rocksdb.RocksDBException; + +public interface ConsumeQueueStoreInterface { + + /** + * Start the consumeQueueStore + */ + void start(); + + /** + * Load from file. + * @return true if loaded successfully. + */ + boolean load(); + + /** + * load after destroy + */ + boolean loadAfterDestroy(); + + /** + * Recover from file. + */ + void recover(); + + /** + * Recover concurrently from file. + * @return true if recovered successfully. + */ + boolean recoverConcurrently(); + + /** + * Shutdown the consumeQueueStore + * @return true if shutdown successfully. + */ + boolean shutdown(); + + /** + * destroy all consumeQueues + */ + void destroy(); + + /** + * destroy the specific consumeQueue + * @throws RocksDBException only in rocksdb mode + */ + void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException; + + /** + * Flush cache to file. + * @param consumeQueue the consumeQueue will be flushed + * @param flushLeastPages the minimum number of pages to be flushed + * @return true if any data has been flushed. + */ + boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages); + + /** + * clean expired data from minPhyOffset + * @param minPhyOffset + */ + void cleanExpired(long minPhyOffset); + + /** + * Check files. + */ + void checkSelf(); + + /** + * Delete expired files ending at min commit log position. + * @param consumeQueue + * @param minCommitLogPos min commit log position + * @return deleted file numbers. + */ + int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos); + + /** + * Is the first file available? + * @param consumeQueue + * @return true if it's available + */ + boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue); + + /** + * Does the first file exist? + * @param consumeQueue + * @return true if it exists + */ + boolean isFirstFileExist(ConsumeQueueInterface consumeQueue); + + /** + * Roll to next file. + * @param consumeQueue + * @param offset next beginning offset + * @return the beginning offset of the next file + */ + long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset); + + /** + * truncate dirty data + * @param offsetToTruncate + * @throws RocksDBException only in rocksdb mode + */ + void truncateDirty(long offsetToTruncate) throws RocksDBException; + + /** + * Apply the dispatched request and build the consume queue. This function should be idempotent. + * + * @param consumeQueue consume queue + * @param request dispatch request + */ + void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request); + + /** + * Apply the dispatched request. This function should be idempotent. + * + * @param request dispatch request + * @throws RocksDBException only in rocksdb mode will throw exception + */ + void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException; + + /** + * range query cqUnit(ByteBuffer) in rocksdb + * @param topic + * @param queueId + * @param startIndex + * @param num + * @return the byteBuffer list of the topic-queueId in rocksdb + * @throws RocksDBException only in rocksdb mode + */ + List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException; + + /** + * query cqUnit(ByteBuffer) in rocksdb + * @param topic + * @param queueId + * @param startIndex + * @return the byteBuffer of the topic-queueId in rocksdb + * @throws RocksDBException only in rocksdb mode + */ + ByteBuffer get(final String topic, final int queueId, final long startIndex) throws RocksDBException; + + /** + * get consumeQueue table + * @return the consumeQueue table + */ + ConcurrentMap> getConsumeQueueTable(); + + /** + * Assign queue offset. + * @param msg message itself + * @throws RocksDBException only in rocksdb mode + */ + void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException; + + /** + * Increase queue offset. + * @param msg message itself + * @param messageNum message number + */ + void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum); + + /** + * recover topicQueue table by minPhyOffset + * @param minPhyOffset + */ + void recoverOffsetTable(long minPhyOffset); + + /** + * set topicQueue table + * @param topicQueueTable + */ + void setTopicQueueTable(ConcurrentMap topicQueueTable); + + /** + * remove topic-queueId from topicQueue table + * @param topic + * @param queueId + */ + void removeTopicQueueTable(String topic, Integer queueId); + + /** + * get topicQueue table + * @return the topicQueue table + */ + ConcurrentMap getTopicQueueTable(); + + /** + * get the max physical offset in consumeQueue + * @param topic + * @param queueId + * @return + */ + Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId); + + /** + * get maxOffset of specific topic-queueId in topicQueue table + * @param topic + * @param queueId + * @return the max offset in QueueOffsetOperator + */ + Long getMaxOffset(String topic, int queueId); + + /** + * get max physic offset in consumeQueue + * @return the max physic offset in consumeQueue + * @throws RocksDBException only in rocksdb mode + */ + long getMaxPhyOffsetInConsumeQueue() throws RocksDBException; + + /** + * get min logic offset of specific topic-queueId in consumeQueue + * @param topic + * @param queueId + * @return the min logic offset of specific topic-queueId in consumeQueue + * @throws RocksDBException only in rocksdb mode + */ + long getMinOffsetInQueue(final String topic, final int queueId) throws RocksDBException; + + /** + * get max logic offset of specific topic-queueId in consumeQueue + * @param topic + * @param queueId + * @return the max logic offset of specific topic-queueId in consumeQueue + * @throws RocksDBException only in rocksdb mode + */ + long getMaxOffsetInQueue(final String topic, final int queueId) throws RocksDBException; + + /** + * Get the message whose timestamp is the smallest, greater than or equal to the given time and when there are more + * than one message satisfy the condition, decide which one to return based on boundaryType. + * @param timestamp timestamp + * @param boundaryType Lower or Upper + * @return the offset(index) + * @throws RocksDBException only in rocksdb mode + */ + long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException; + + /** + * find or create the consumeQueue + * @param topic + * @param queueId + * @return the consumeQueue + */ + ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId); + + /** + * find the consumeQueueMap of topic + * @param topic + * @return the consumeQueueMap of topic + */ + ConcurrentMap findConsumeQueueMap(String topic); + + /** + * get the total size of all consumeQueue + * @return the total size of all consumeQueue + */ + long getTotalSize(); + + /** + * Get store time from commitlog by cqUnit + * @param cqUnit + * @return + */ + long getStoreTime(CqUnit cqUnit); +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java new file mode 100644 index 00000000000..d6291d90879 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.config.MessageStoreConfig; + +public class MultiDispatch { + + public static String lmqQueueKey(String queueName) { + StringBuilder keyBuilder = new StringBuilder(); + keyBuilder.append(queueName); + keyBuilder.append('-'); + int queueId = 0; + keyBuilder.append(queueId); + return keyBuilder.toString(); + } + + public static boolean isNeedHandleMultiDispatch(MessageStoreConfig messageStoreConfig, String topic) { + return messageStoreConfig.isEnableMultiDispatch() + && !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) + && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX) + && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); + } + + public static boolean checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, DispatchRequest dispatchRequest) { + if (!isNeedHandleMultiDispatch(messageStoreConfig, dispatchRequest.getTopic())) { + return false; + } + Map prop = dispatchRequest.getPropertiesMap(); + if (prop == null || prop.isEmpty()) { + return false; + } + String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); + if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { + return false; + } + return true; + } + + public static List checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, List dispatchRequests) { + if (!messageStoreConfig.isEnableMultiDispatch() || dispatchRequests == null || dispatchRequests.size() == 0) { + return null; + } + List result = new ArrayList<>(); + for (DispatchRequest dispatchRequest : dispatchRequests) { + if (checkMultiDispatchQueue(messageStoreConfig, dispatchRequest)) { + result.add(dispatchRequest); + } + } + return dispatchRequests; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java index 2545bbf523d..8da3748281f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java @@ -41,6 +41,10 @@ public long getQueueOffset(String topicQueueKey) { return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); } + public Long getTopicQueueNextOffset(String topicQueueKey) { + return this.topicQueueTable.get(topicQueueKey); + } + public void increaseQueueOffset(String topicQueueKey, short messageNum) { Long queueOffset = ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); topicQueueTable.put(topicQueueKey, queueOffset + messageNum); @@ -63,6 +67,10 @@ public long getLmqOffset(String topicQueueKey) { return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); } + public Long getLmqTopicQueueNextOffset(String topicQueueKey) { + return this.lmqTopicQueueTable.get(topicQueueKey); + } + public void increaseLmqOffset(String topicQueueKey, short messageNum) { Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); this.lmqTopicQueueTable.put(topicQueueKey, lmqOffset + messageNum); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java new file mode 100644 index 00000000000..759be395d56 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.attribute.CQType; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.ConsumeQueue; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.MessageFilter; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.RocksDBException; + +public class RocksDBConsumeQueue implements ConsumeQueueInterface { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + + private final MessageStore messageStore; + private final String topic; + private final int queueId; + + public RocksDBConsumeQueue(final MessageStore messageStore, final String topic, final int queueId) { + this.messageStore = messageStore; + this.topic = topic; + this.queueId = queueId; + } + + public RocksDBConsumeQueue(final String topic, final int queueId) { + this.messageStore = null; + this.topic = topic; + this.queueId = queueId; + } + + @Override + public boolean load() { + return true; + } + + @Override + public void recover() { + // ignore + } + + @Override + public void checkSelf() { + // ignore + } + + @Override + public boolean flush(final int flushLeastPages) { + return true; + } + + @Override + public void destroy() { + // ignore + } + + @Override + public void truncateDirtyLogicFiles(long maxCommitLogPos) { + // ignored + } + + @Override + public int deleteExpiredFile(long minCommitLogPos) { + return 0; + } + + @Override + public long rollNextFile(long nextBeginOffset) { + return 0; + } + + @Override + public boolean isFirstFileAvailable() { + return true; + } + + @Override + public boolean isFirstFileExist() { + return true; + } + + @Override + public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) { + // ignore + } + + @Override + public void cleanSwappedMap(long forceCleanSwapIntervalMs) { + // ignore + } + + @Override + public long getMaxOffsetInQueue() { + try { + return this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); + } catch (RocksDBException e) { + ERROR_LOG.error("getMaxOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e); + return 0; + } + } + + @Override + public long getMessageTotalInQueue() { + try { + long maxOffsetInQueue = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); + long minOffsetInQueue = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId); + return maxOffsetInQueue - minOffsetInQueue; + } catch (RocksDBException e) { + ERROR_LOG.error("getMessageTotalInQueue Failed. topic: {}, queueId: {}, {}", topic, queueId, e); + } + return -1; + } + + /** + * We already implement it in RocksDBConsumeQueueStore. + * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime + * @param timestamp timestamp + * @return + */ + @Override + public long getOffsetInQueueByTime(long timestamp) { + return 0; + } + + /** + * We already implement it in RocksDBConsumeQueueStore. + * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime + * @param timestamp timestamp + * @param boundaryType Lower or Upper + * @return + */ + @Override + public long getOffsetInQueueByTime(long timestamp, BoundaryType boundaryType) { + return 0; + } + + @Override + public long getMaxPhysicOffset() { + Long maxPhyOffset = this.messageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(topic, queueId); + return maxPhyOffset == null ? -1 : maxPhyOffset; + } + + @Override + public long getMinLogicOffset() { + return 0; + } + + @Override + public CQType getCQType() { + return CQType.RocksDBCQ; + } + + @Override + public long getTotalSize() { + // ignored + return 0; + } + + @Override + public int getUnitSize() { + // attention: unitSize should equal to 'ConsumeQueue.CQ_STORE_UNIT_SIZE' + return ConsumeQueue.CQ_STORE_UNIT_SIZE; + } + + /** + * Ignored, we already implement this method + * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) + */ + @Override + public void correctMinOffset(long minCommitLogOffset) { + + } + + /** + * Ignored, in rocksdb mode, we build cq in RocksDBConsumeQueueStore + * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore#putMessagePosition() + */ + @Override + public void putMessagePositionInfoWrapper(DispatchRequest request) { + + } + + @Override + public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg) throws RocksDBException { + String topicQueueKey = getTopic() + "-" + getQueueId(); + Long queueOffset = queueOffsetOperator.getTopicQueueNextOffset(topicQueueKey); + if (queueOffset == null) { + // we will recover topic queue table from rocksdb when we use it. + queueOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); + queueOffsetOperator.updateQueueOffset(topicQueueKey, queueOffset); + } + msg.setQueueOffset(queueOffset); + + // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), + // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. + if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { + return; + } + String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + if (StringUtils.isBlank(multiDispatchQueue)) { + return; + } + String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + Long[] queueOffsets = new Long[queues.length]; + for (int i = 0; i < queues.length; i++) { + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { + String key = MultiDispatch.lmqQueueKey(queues[i]); + queueOffsets[i] = queueOffsetOperator.getLmqTopicQueueNextOffset(key); + } + } + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, + StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); + msg.removeWaitStorePropertyString(); + } + + @Override + public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg, short messageNum) { + String topicQueueKey = getTopic() + "-" + getQueueId(); + queueOffsetOperator.increaseQueueOffset(topicQueueKey, messageNum); + + // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), + // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. + if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { + return; + } + String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + if (StringUtils.isBlank(multiDispatchQueue)) { + return; + } + String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + for (int i = 0; i < queues.length; i++) { + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { + String key = MultiDispatch.lmqQueueKey(queues[i]); + queueOffsetOperator.increaseLmqOffset(key, (short) 1); + } + } + } + + @Override + public long estimateMessageCount(long from, long to, MessageFilter filter) { + // todo + return 0; + } + + @Override + public long getMinOffsetInQueue() { + return this.messageStore.getMinOffsetInQueue(this.topic, this.queueId); + } + + private int pullNum(long cqOffset, long maxCqOffset) { + long diffLong = maxCqOffset - cqOffset; + if (diffLong < Integer.MAX_VALUE) { + int diffInt = (int) diffLong; + return diffInt > 16 ? 16 : diffInt; + } + return 16; + } + + @Override + public ReferredIterator iterateFrom(final long startIndex) { + try { + long maxCqOffset = getMaxOffsetInQueue(); + if (startIndex < maxCqOffset) { + int num = pullNum(startIndex, maxCqOffset); + return iterateFrom0(startIndex, num); + } + } catch (RocksDBException e) { + log.error("[RocksDBConsumeQueue] iterateFrom error!", e); + } + return null; + } + + @Override + public ReferredIterator iterateFrom(long startIndex, int count) throws RocksDBException { + long maxCqOffset = getMaxOffsetInQueue(); + if (startIndex < maxCqOffset) { + int num = Math.min((int)(maxCqOffset - startIndex), count); + return iterateFrom0(startIndex, num); + } + return null; + } + + @Override + public CqUnit get(long index) { + Pair pair = getCqUnitAndStoreTime(index); + return pair == null ? null : pair.getObject1(); + } + + @Override + public Pair getCqUnitAndStoreTime(long index) { + ByteBuffer byteBuffer; + try { + byteBuffer = this.messageStore.getQueueStore().get(topic, queueId, index); + } catch (RocksDBException e) { + ERROR_LOG.error("getUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e); + return null; + } + if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) { + return null; + } + long phyOffset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + long tagCode = byteBuffer.getLong(); + long messageStoreTime = byteBuffer.getLong(); + return new Pair<>(new CqUnit(index, phyOffset, size, tagCode), messageStoreTime); + } + + @Override + public Pair getEarliestUnitAndStoreTime() { + try { + long minOffset = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId); + return getCqUnitAndStoreTime(minOffset); + } catch (RocksDBException e) { + ERROR_LOG.error("getEarliestUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e); + } + return null; + } + + @Override + public CqUnit getEarliestUnit() { + Pair pair = getEarliestUnitAndStoreTime(); + return pair == null ? null : pair.getObject1(); + } + + @Override + public CqUnit getLatestUnit() { + try { + long maxOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); + return get(maxOffset); + } catch (RocksDBException e) { + ERROR_LOG.error("getLatestUnit Failed. topic: {}, queueId: {}, {}", topic, queueId, e.getMessage()); + } + return null; + } + + @Override + public long getLastOffset() { + return getMaxPhysicOffset(); + } + + private ReferredIterator iterateFrom0(final long startIndex, final int count) throws RocksDBException { + List byteBufferList = this.messageStore.getQueueStore().rangeQuery(topic, queueId, startIndex, count); + if (byteBufferList == null || byteBufferList.isEmpty()) { + if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + log.warn("iterateFrom0 - find nothing, startIndex:{}, count:{}", startIndex, count); + } + return null; + } + return new RocksDBConsumeQueueIterator(byteBufferList, startIndex); + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public int getQueueId() { + return queueId; + } + + private class RocksDBConsumeQueueIterator implements ReferredIterator { + private final List byteBufferList; + private final long startIndex; + private final int totalCount; + private int currentIndex; + + public RocksDBConsumeQueueIterator(final List byteBufferList, final long startIndex) { + this.byteBufferList = byteBufferList; + this.startIndex = startIndex; + this.totalCount = byteBufferList.size(); + this.currentIndex = 0; + } + + @Override + public boolean hasNext() { + return this.currentIndex < this.totalCount; + } + + @Override + public CqUnit next() { + if (!hasNext()) { + return null; + } + final int currentIndex = this.currentIndex; + final ByteBuffer byteBuffer = this.byteBufferList.get(currentIndex); + CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong()); + this.currentIndex++; + return cqUnit; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove"); + } + + @Override + public void release() { + } + + @Override + public CqUnit nextAndRelease() { + try { + return next(); + } finally { + release(); + } + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java new file mode 100644 index 00000000000..6fa66282e9d --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java @@ -0,0 +1,641 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.ConsumeQueue; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; + +import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; + +public class RocksDBConsumeQueueOffsetTable { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + + private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8); + private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8); + + /** + * Rocksdb ConsumeQueue's Offset unit. Format: + * + * 
+     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬─────────────┐
+     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  Max(Min) │  CTRL_1   │   QueueId   │
+     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │  (4 Bytes)  │
+     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴─────────────┤
+     * │                                                    Key Unit                                                   │
+     * │                                                                                                               │
+     *
+ * + * 
+     * ┌─────────────────────────────┬────────────────────────┐
+     * │  CommitLog Physical Offset  │   ConsumeQueue Offset  │
+     * │        (8 Bytes)            │    (8 Bytes)           │
+     * ├─────────────────────────────┴────────────────────────┤
+     * │                     Value Unit                       │
+     * │                                                      │
+     *
+ * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) = 16 Bytes + */ + private static final int OFFSET_PHY_OFFSET = 0; + private static final int OFFSET_CQ_OFFSET = 8; + /** + * + * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐ + * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │ + * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │ + * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴─────────────┤ + */ + private static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4; + private static final int OFFSET_VALUE_LENGTH = 8 + 8; + + /** + * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread. + * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread. + * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of + * RocksDBConsumeQueueOffsetTable to find it. + */ + private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC; + private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8); + private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length; + private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN); + private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN]; + private final ByteBuffer maxPhyOffsetBB; + static { + buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true); + INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); + INNER_CHECKPOINT_TOPIC.get(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY); + } + + private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; + private final ConsumeQueueRocksDBStorage rocksDBStorage; + private final DefaultMessageStore messageStore; + + private ColumnFamilyHandle offsetCFH; + + /** + * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them + * from heap to avoid accessing rocksdb. + * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset --> topicQueueMaxCqOffset + * @see ConsumeQueue#getMinLogicOffset(), minLogicOffset --> topicQueueMinOffset + */ + private final Map topicQueueMinOffset; + private final Map topicQueueMaxCqOffset; + + public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable, + ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { + this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable; + this.rocksDBStorage = rocksDBStorage; + this.messageStore = messageStore; + this.topicQueueMinOffset = new ConcurrentHashMap(1024); + this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024); + + this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8); + } + + public void load() { + this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle(); + } + + public void updateTempTopicQueueMaxOffset(final Pair offsetBBPair, + final byte[] topicBytes, final DispatchRequest request, + final Map> tempTopicQueueMaxOffsetMap) { + buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request); + ByteBuffer topicQueueId = offsetBBPair.getObject1(); + ByteBuffer maxOffsetBB = offsetBBPair.getObject2(); + Pair old = tempTopicQueueMaxOffsetMap.get(topicQueueId); + if (old == null) { + tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request)); + } else { + long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET); + long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET); + if (maxOffset >= oldMaxOffset) { + ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset); + } + } + } + + public void putMaxPhyAndCqOffset(final Map> tempTopicQueueMaxOffsetMap, + final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { + for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { + writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1()); + } + + appendMaxPhyOffset(writeBatch, maxPhyOffset); + } + + public void putHeapMaxCqOffset(final Map> tempTopicQueueMaxOffsetMap) { + for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { + DispatchRequest request = entry.getValue().getObject2(); + putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset()); + } + } + + /** + * When topic is deleted, we clean up its offset info in rocksdb. + * @param topic + * @param queueId + * @throws RocksDBException + */ + public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false); + byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array()); + Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; + + final ByteBuffer maxOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, true); + byte[] maxOffsetBytes = this.rocksDBStorage.getOffset(maxOffsetKey.array()); + Long endCQOffset = (maxOffsetBytes != null) ? ByteBuffer.wrap(maxOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; + + writeBatch.delete(this.offsetCFH, minOffsetKey.array()); + writeBatch.delete(this.offsetCFH, maxOffsetKey.array()); + + String topicQueueId = buildTopicQueueId(topic, queueId); + removeHeapMinCqOffset(topicQueueId); + removeHeapMaxCqOffset(topicQueueId); + + log.info("RocksDB offset table delete topic: {}, queueId: {}, minOffset: {}, maxOffset: {}", topic, queueId, + startCQOffset, endCQOffset); + } + + private void appendMaxPhyOffset(final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { + final ByteBuffer maxPhyOffsetBB = this.maxPhyOffsetBB; + maxPhyOffsetBB.position(0).limit(8); + maxPhyOffsetBB.putLong(maxPhyOffset); + maxPhyOffsetBB.flip(); + + INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); + writeBatch.put(this.offsetCFH, INNER_CHECKPOINT_TOPIC, maxPhyOffsetBB); + } + + public long getMaxPhyOffset() throws RocksDBException { + byte[] valueBytes = this.rocksDBStorage.getOffset(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY); + if (valueBytes == null) { + return 0; + } + ByteBuffer valueBB = ByteBuffer.wrap(valueBytes); + return valueBB.getLong(0); + } + + /** + * Traverse the offset table to find dirty topic + * @param existTopicSet + * @return + */ + public Map> iterateOffsetTable2FindDirty(final Set existTopicSet) { + Map> topicQueueIdToBeDeletedMap = new HashMap<>(); + + RocksIterator iterator = null; + try { + iterator = rocksDBStorage.seekOffsetCF(); + if (iterator == null) { + return topicQueueIdToBeDeletedMap; + } + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + byte[] key = iterator.key(); + byte[] value = iterator.value(); + if (key == null || key.length <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + || value == null || value.length != OFFSET_VALUE_LENGTH) { + continue; + } + ByteBuffer keyBB = ByteBuffer.wrap(key); + int topicLen = keyBB.getInt(0); + byte[] topicBytes = new byte[topicLen]; + /** + * "Topic Bytes Array Size" + "CTRL_1" = 4 + 1 + */ + keyBB.position(4 + 1); + keyBB.get(topicBytes); + String topic = new String(topicBytes, CHARSET_UTF8); + if (TopicValidator.isSystemTopic(topic)) { + continue; + } + + /** + * "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1" + "Max(min)" + "CTRL_1" + * = 4 + 1 + topicLen + 1 + 3 + 1 + */ + int queueId = keyBB.getInt(4 + 1 + topicLen + 1 + 3 + 1); + + if (!existTopicSet.contains(topic)) { + ByteBuffer valueBB = ByteBuffer.wrap(value); + long cqOffset = valueBB.getLong(OFFSET_CQ_OFFSET); + + Set topicQueueIdSet = topicQueueIdToBeDeletedMap.get(topic); + if (topicQueueIdSet == null) { + Set newSet = new HashSet<>(); + newSet.add(queueId); + topicQueueIdToBeDeletedMap.put(topic, newSet); + } else { + topicQueueIdSet.add(queueId); + } + ERROR_LOG.info("RocksDBConsumeQueueOffsetTable has dirty cqOffset. topic: {}, queueId: {}, cqOffset: {}", + topic, queueId, cqOffset); + } + } + } catch (Exception e) { + ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e); + } finally { + if (iterator != null) { + iterator.close(); + } + } + return topicQueueIdToBeDeletedMap; + } + + public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException { + Long maxCqOffset = getHeapMaxCqOffset(topic, queueId); + + if (maxCqOffset == null) { + final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); + maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null; + String topicQueueId = buildTopicQueueId(topic, queueId); + this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L); + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset); + } + } + + return maxCqOffset; + } + + /** + * truncate dirty offset in rocksdb + * @param offsetToTruncate + * @throws RocksDBException + */ + public void truncateDirty(long offsetToTruncate) throws RocksDBException { + correctMaxPyhOffset(offsetToTruncate); + + ConcurrentMap allTopicConfigMap = this.messageStore.getTopicConfigs(); + if (allTopicConfigMap == null) { + return; + } + for (TopicConfig topicConfig : allTopicConfigMap.values()) { + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + truncateDirtyOffset(topicConfig.getTopicName(), i); + } + } + } + + private Pair isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException { + PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); + if (phyAndCQOffset != null) { + final long phyOffset = phyAndCQOffset.getPhyOffset(); + final long cqOffset = phyAndCQOffset.getCqOffset(); + + return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset); + } + ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId); + if (byteBuffer == null) { + return new Pair(false, 0L); + } + final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); + final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); + if (phyOffset >= minPhyOffset) { + String topicQueueId = buildTopicQueueId(topic, queueId); + PhyAndCQOffset newPhyAndCQOffset = new PhyAndCQOffset(phyOffset, cqOffset); + this.topicQueueMinOffset.putIfAbsent(topicQueueId, newPhyAndCQOffset); + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset); + } + return new Pair(true, cqOffset); + } + return new Pair(false, cqOffset); + } + + private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException { + final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); + if (byteBuffer == null) { + return; + } + + long maxPhyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); + long maxCqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); + long maxPhyOffsetInCQ = getMaxPhyOffset(); + + if (maxPhyOffset >= maxPhyOffsetInCQ) { + correctMaxCqOffset(topic, queueId, maxCqOffset, maxPhyOffsetInCQ); + Long newMaxCqOffset = getHeapMaxCqOffset(topic, queueId); + ROCKSDB_LOG.warn("truncateDirtyLogicFile topic: {}, queueId: {} from {} to {}", topic, queueId, + maxPhyOffset, newMaxCqOffset); + } + } + + private void correctMaxPyhOffset(long maxPhyOffset) throws RocksDBException { + if (!this.rocksDBStorage.hold()) { + return; + } + try { + WriteBatch writeBatch = new WriteBatch(); + long oldMaxPhyOffset = getMaxPhyOffset(); + if (oldMaxPhyOffset <= maxPhyOffset) { + return; + } + log.info("correctMaxPyhOffset, oldMaxPhyOffset={}, newMaxPhyOffset={}", oldMaxPhyOffset, maxPhyOffset); + appendMaxPhyOffset(writeBatch, maxPhyOffset); + this.rocksDBStorage.batchPut(writeBatch); + } catch (RocksDBException e) { + ERROR_LOG.error("correctMaxPyhOffset Failed.", e); + throw e; + } finally { + this.rocksDBStorage.release(); + } + } + + public long getMinCqOffset(String topic, int queueId) throws RocksDBException { + final long minPhyOffset = this.messageStore.getMinPhyOffset(); + Pair pair = isMinOffsetOk(topic, queueId, minPhyOffset); + final long cqOffset = pair.getObject2(); + if (!pair.getObject1() && correctMinCqOffset(topic, queueId, cqOffset, minPhyOffset)) { + PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); + if (phyAndCQOffset != null) { + if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("getMinOffsetInQueue miss heap. topic: {}, queueId: {}, old: {}, new: {}", + topic, queueId, cqOffset, phyAndCQOffset); + } + return phyAndCQOffset.getCqOffset(); + } + } + return cqOffset; + } + + public Long getMaxPhyOffset(String topic, int queueId) { + try { + ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); + if (byteBuffer != null) { + return byteBuffer.getLong(OFFSET_PHY_OFFSET); + } + } catch (Exception e) { + ERROR_LOG.info("getMaxPhyOffset error. topic: {}, queueId: {}", topic, queueId); + } + return null; + } + + private ByteBuffer getMinPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException { + return getPhyAndCqOffsetInKV(topic, queueId, false); + } + + private ByteBuffer getMaxPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException { + return getPhyAndCqOffsetInKV(topic, queueId, true); + } + + private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max); + + byte[] value = this.rocksDBStorage.getOffset(keyBB.array()); + return (value != null) ? ByteBuffer.wrap(value) : null; + } + + private String buildTopicQueueId(final String topic, final int queueId) { + return topic + "-" + queueId; + } + + private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) { + String topicQueueId = buildTopicQueueId(topic, queueId); + PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset); + this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset); + } + + private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) { + String topicQueueId = buildTopicQueueId(topic, queueId); + Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset); + if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) { + ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset); + } + } + + private PhyAndCQOffset getHeapMinOffset(final String topic, final int queueId) { + return this.topicQueueMinOffset.get(buildTopicQueueId(topic, queueId)); + } + + private Long getHeapMaxCqOffset(final String topic, final int queueId) { + String topicQueueId = buildTopicQueueId(topic, queueId); + return this.topicQueueMaxCqOffset.get(topicQueueId); + } + + private PhyAndCQOffset removeHeapMinCqOffset(String topicQueueId) { + return this.topicQueueMinOffset.remove(topicQueueId); + } + + private Long removeHeapMaxCqOffset(String topicQueueId) { + return this.topicQueueMaxCqOffset.remove(topicQueueId); + } + + private void updateCqOffset(final String topic, final int queueId, final long phyOffset, + final long cqOffset, boolean max) throws RocksDBException { + if (!this.rocksDBStorage.hold()) { + return; + } + WriteBatch writeBatch = new WriteBatch(); + try { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max); + + final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset); + writeBatch.put(this.offsetCFH, offsetKey.array(), offsetValue.array()); + this.rocksDBStorage.batchPut(writeBatch); + + if (max) { + putHeapMaxCqOffset(topic, queueId, cqOffset); + } else { + putHeapMinCqOffset(topic, queueId, phyOffset, cqOffset); + } + } catch (RocksDBException e) { + ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e); + throw e; + } finally { + writeBatch.close(); + this.rocksDBStorage.release(); + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}", + max ? "max" : "min", topic, queueId, phyOffset, cqOffset); + } + } + } + + private boolean correctMaxCqOffset(final String topic, final int queueId, final long maxCQOffset, + final long maxPhyOffsetInCQ) throws RocksDBException { + // 'getMinOffsetInQueue' may correct minCqOffset and put it into heap + long minCQOffset = getMinCqOffset(topic, queueId); + PhyAndCQOffset minPhyAndCQOffset = getHeapMinOffset(topic, queueId); + if (minPhyAndCQOffset == null + || minPhyAndCQOffset.getCqOffset() != minCQOffset + || minPhyAndCQOffset.getPhyOffset() > maxPhyOffsetInCQ) { + ROCKSDB_LOG.info("[BUG] correctMaxCqOffset error! topic: {}, queueId: {}, maxPhyOffsetInCQ: {}, " + + "minCqOffset: {}, phyAndCQOffset: {}", + topic, queueId, maxPhyOffsetInCQ, minCQOffset, minPhyAndCQOffset); + throw new RocksDBException("correctMaxCqOffset error"); + } + + long high = maxCQOffset; + long low = minCQOffset; + PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, + low, maxPhyOffsetInCQ, false); + + long targetCQOffset = targetPhyAndCQOffset.getCqOffset(); + long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset(); + + if (targetCQOffset == -1) { + if (maxCQOffset != minCQOffset) { + updateCqOffset(topic, queueId, minPhyAndCQOffset.getPhyOffset(), minCQOffset, true); + } + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyAndCQOffset.getPhyOffset()); + } + return false; + } else { + updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, true); + return true; + } + } + + private boolean correctMinCqOffset(final String topic, final int queueId, + final long minCQOffset, final long minPhyOffset) throws RocksDBException { + final ByteBuffer maxBB = getMaxPhyAndCqOffsetInKV(topic, queueId); + if (maxBB == null) { + updateCqOffset(topic, queueId, minPhyOffset, 0L, false); + return true; + } + final long maxPhyOffset = maxBB.getLong(OFFSET_PHY_OFFSET); + final long maxCQOffset = maxBB.getLong(OFFSET_CQ_OFFSET); + + if (maxPhyOffset < minPhyOffset) { + updateCqOffset(topic, queueId, minPhyOffset, maxCQOffset + 1, false); + return true; + } + + long high = maxCQOffset; + long low = minCQOffset; + PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low, + minPhyOffset, true); + long targetCQOffset = phyAndCQOffset.getCqOffset(); + long targetPhyOffset = phyAndCQOffset.getPhyOffset(); + + if (targetCQOffset == -1) { + if (maxCQOffset != minCQOffset) { + updateCqOffset(topic, queueId, maxPhyOffset, maxCQOffset, false); + } + if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyOffset); + } + return false; + } else { + updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, false); + return true; + } + } + + public static Pair getOffsetByteBufferPair() { + ByteBuffer offsetKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN); + ByteBuffer offsetValue = ByteBuffer.allocateDirect(OFFSET_VALUE_LENGTH); + return new Pair<>(offsetKey, offsetValue); + } + + private void buildOffsetKeyAndValueByteBuffer(final Pair offsetBBPair, + final byte[] topicBytes, final DispatchRequest request) { + final ByteBuffer offsetKey = offsetBBPair.getObject1(); + buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true); + + final ByteBuffer offsetValue = offsetBBPair.getObject2(); + buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset()); + } + + private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) { + ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); + return byteBuffer; + } + + private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) { + byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); + } + + private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, + final boolean max) { + byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1); + if (max) { + byteBuffer.put(MAX_BYTES); + } else { + byteBuffer.put(MIN_BYTES); + } + byteBuffer.put(CTRL_1).putInt(queueId); + byteBuffer.flip(); + } + + private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { + byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH); + buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); + } + + private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH); + buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); + return byteBuffer; + } + + private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { + byteBuffer.putLong(phyOffset).putLong(cqOffset); + byteBuffer.flip(); + } + + static class PhyAndCQOffset { + private final long phyOffset; + private final long cqOffset; + + public PhyAndCQOffset(final long phyOffset, final long cqOffset) { + this.phyOffset = phyOffset; + this.cqOffset = cqOffset; + } + + public long getPhyOffset() { + return this.phyOffset; + } + + public long getCqOffset() { + return this.cqOffset; + } + + @Override + public String toString() { + return "[cqOffset=" + cqOffset + ", phyOffset=" + phyOffset + "]"; + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java new file mode 100644 index 00000000000..78456cfcd85 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; + +public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { + private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + + public static final byte CTRL_0 = '\u0000'; + public static final byte CTRL_1 = '\u0001'; + public static final byte CTRL_2 = '\u0002'; + + private static final int BATCH_SIZE = 16; + public static final int MAX_KEY_LEN = 300; + + private final ScheduledExecutorService scheduledExecutorService; + private final String storePath; + + /** + * we use two tables with different ColumnFamilyHandle, called RocksDBConsumeQueueTable and RocksDBConsumeQueueOffsetTable. + * 1.RocksDBConsumeQueueTable uses to store CqUnit[physicalOffset, msgSize, tagHashCode, msgStoreTime] + * 2.RocksDBConsumeQueueOffsetTable uses to store physicalOffset and consumeQueueOffset(@see PhyAndCQOffset) of topic-queueId + */ + private final ConsumeQueueRocksDBStorage rocksDBStorage; + private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; + private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable; + + private final WriteBatch writeBatch; + private final List bufferDRList; + private final List> cqBBPairList; + private final List> offsetBBPairList; + private final Map> tempTopicQueueMaxOffsetMap; + private volatile boolean isCQError = false; + + public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { + super(messageStore); + + this.storePath = StorePathConfigHelper.getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir()); + this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath, 4); + this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore); + this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); + + this.writeBatch = new WriteBatch(); + this.bufferDRList = new ArrayList(BATCH_SIZE); + this.cqBBPairList = new ArrayList(BATCH_SIZE); + this.offsetBBPairList = new ArrayList(BATCH_SIZE); + for (int i = 0; i < BATCH_SIZE; i++) { + this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); + this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); + } + + this.tempTopicQueueMaxOffsetMap = new HashMap<>(); + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity())); + } + + @Override + public void start() { + log.info("RocksDB ConsumeQueueStore start!"); + this.scheduledExecutorService.scheduleAtFixedRate(() -> { + this.rocksDBStorage.statRocksdb(ROCKSDB_LOG); + }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS); + + this.scheduledExecutorService.scheduleWithFixedDelay(() -> { + cleanDirty(messageStore.getTopicConfigs().keySet()); + }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES); + } + + private void cleanDirty(final Set existTopicSet) { + try { + Map> topicQueueIdToBeDeletedMap = + this.rocksDBConsumeQueueOffsetTable.iterateOffsetTable2FindDirty(existTopicSet); + + for (Map.Entry> entry : topicQueueIdToBeDeletedMap.entrySet()) { + String topic = entry.getKey(); + for (int queueId : entry.getValue()) { + destroy(new RocksDBConsumeQueue(topic, queueId)); + } + } + } catch (Exception e) { + log.error("cleanUnusedTopic Failed.", e); + } + } + + @Override + public boolean load() { + boolean result = this.rocksDBStorage.start(); + this.rocksDBConsumeQueueTable.load(); + this.rocksDBConsumeQueueOffsetTable.load(); + log.info("load rocksdb consume queue {}.", result ? "OK" : "Failed"); + return result; + } + + @Override + public boolean loadAfterDestroy() { + return this.load(); + } + + @Override + public void recover() { + // ignored + } + + @Override + public boolean recoverConcurrently() { + return true; + } + + @Override + public boolean shutdown() { + this.scheduledExecutorService.shutdown(); + return shutdownInner(); + } + + private boolean shutdownInner() { + return this.rocksDBStorage.shutdown(); + } + + @Override + public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { + if (request == null || this.bufferDRList.size() >= BATCH_SIZE) { + putMessagePosition(); + } + if (request != null) { + this.bufferDRList.add(request); + } + } + + public void putMessagePosition() throws RocksDBException { + final int maxRetries = 30; + for (int i = 0; i < maxRetries; i++) { + if (putMessagePosition0()) { + if (this.isCQError) { + this.messageStore.getRunningFlags().clearLogicsQueueError(); + this.isCQError = false; + } + return; + } else { + ERROR_LOG.warn("{} put cq Failed. retryTime: {}", i); + try { + Thread.sleep(100); + } catch (InterruptedException ignored) { + } + } + } + if (!this.isCQError) { + ERROR_LOG.error("[BUG] put CQ Failed."); + this.messageStore.getRunningFlags().makeLogicsQueueError(); + this.isCQError = true; + } + throw new RocksDBException("put CQ Failed"); + } + + private boolean putMessagePosition0() { + if (!this.rocksDBStorage.hold()) { + return false; + } + + final Map> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap; + try { + final List bufferDRList = this.bufferDRList; + final int size = bufferDRList.size(); + if (size == 0) { + return true; + } + final List> cqBBPairList = this.cqBBPairList; + final List> offsetBBPairList = this.offsetBBPairList; + final WriteBatch writeBatch = this.writeBatch; + + long maxPhyOffset = 0; + for (int i = size - 1; i >= 0; i--) { + final DispatchRequest request = bufferDRList.get(i); + final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8); + + this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch); + this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i), + topicBytes, request, tempTopicQueueMaxOffsetMap); + + final int msgSize = request.getMsgSize(); + final long phyOffset = request.getCommitLogOffset(); + if (phyOffset + msgSize >= maxPhyOffset) { + maxPhyOffset = phyOffset + msgSize; + } + } + + this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset); + + // clear writeBatch in batchPut + this.rocksDBStorage.batchPut(writeBatch); + + this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap); + + long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp(); + if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE + || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) { + this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp); + } + this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp); + + notifyMessageArriveAndClear(); + return true; + } catch (Exception e) { + ERROR_LOG.error("putMessagePosition0 Failed.", e); + return false; + } finally { + tempTopicQueueMaxOffsetMap.clear(); + this.rocksDBStorage.release(); + } + } + + private void notifyMessageArriveAndClear() { + final List bufferDRList = this.bufferDRList; + try { + for (DispatchRequest dp : bufferDRList) { + this.messageStore.notifyMessageArriveIfNecessary(dp); + } + } catch (Exception e) { + ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e); + } finally { + bufferDRList.clear(); + } + } + + @Override + public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { + return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num); + } + + @Override + public ByteBuffer get(final String topic, final int queueId, final long cqOffset) throws RocksDBException { + return this.rocksDBConsumeQueueTable.getCQInKV(topic, queueId, cqOffset); + } + + /** + * Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them + * when we use them, we call it lazy correction. + * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short) + * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) + */ + @Override + public void recoverOffsetTable(long minPhyOffset) { + + } + + @Override + public void destroy() { + try { + shutdownInner(); + FileUtils.deleteDirectory(new File(this.storePath)); + } catch (Exception e) { + ERROR_LOG.error("destroy cq Failed. {}", this.storePath, e); + } + } + + @Override + public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException { + String topic = consumeQueue.getTopic(); + int queueId = consumeQueue.getQueueId(); + if (StringUtils.isEmpty(topic) || queueId < 0 || !this.rocksDBStorage.hold()) { + return; + } + + WriteBatch writeBatch = new WriteBatch(); + try { + this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch); + this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch); + + this.rocksDBStorage.batchPut(writeBatch); + } catch (RocksDBException e) { + ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e); + throw e; + } finally { + writeBatch.close(); + this.rocksDBStorage.release(); + } + } + + @Override + public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { + try { + this.rocksDBStorage.flushWAL(); + } catch (Exception e) { + } + return true; + } + + @Override + public void checkSelf() { + // ignored + } + + @Override + public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) { + // ignored + return 0; + } + + /** + * We do not need to truncate dirty CQ in RocksDBConsumeQueueTable, Because dirty CQ in RocksDBConsumeQueueTable + * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted. + * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in + * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset). + * @param offsetToTruncate + * @throws RocksDBException + */ + @Override + public void truncateDirty(long offsetToTruncate) throws RocksDBException { + long maxPhyOffsetInRocksdb = getMaxPhyOffsetInConsumeQueue(); + if (offsetToTruncate >= maxPhyOffsetInRocksdb) { + return; + } + + this.rocksDBConsumeQueueOffsetTable.truncateDirty(offsetToTruncate); + } + + @Override + public void cleanExpired(final long minPhyOffset) { + this.rocksDBStorage.manualCompaction(minPhyOffset); + } + + @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException { + final long minPhysicOffset = this.messageStore.getMinPhyOffset(); + long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); + Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); + if (high == null || high == -1) { + return 0; + } + return this.rocksDBConsumeQueueTable.binarySearchInCQByTime(topic, queueId, high, low, timestamp, minPhysicOffset); + } + + @Override + public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException { + Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); + return (maxOffset != null) ? maxOffset + 1 : 0; + } + + @Override + public long getMinOffsetInQueue(String topic, int queueId) throws RocksDBException { + return this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); + } + + @Override + public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) { + return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(topic, queueId); + } + + @Override + public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException { + return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(); + } + + @Override + public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { + ConcurrentMap map = this.consumeQueueTable.get(topic); + if (null == map) { + ConcurrentMap newMap = new ConcurrentHashMap<>(128); + ConcurrentMap oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap); + if (oldMap != null) { + map = oldMap; + } else { + map = newMap; + } + } + + ConsumeQueueInterface logic = map.get(queueId); + if (logic != null) { + return logic; + } + + ConsumeQueueInterface newLogic = new RocksDBConsumeQueue(this.messageStore, topic, queueId); + ConsumeQueueInterface oldLogic = map.putIfAbsent(queueId, newLogic); + + return oldLogic != null ? oldLogic : newLogic; + } + + @Override + public long rollNextFile(ConsumeQueueInterface consumeQueue, long offset) { + return 0; + } + + @Override + public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) { + return true; + } + + @Override + public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { + return true; + } + + @Override + public long getTotalSize() { + return 0; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java new file mode 100644 index 00000000000..0a735ea27c1 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset; +import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; + +import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0; +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2; + +/** + * We use RocksDBConsumeQueueTable to store cqUnit. + */ +public class RocksDBConsumeQueueTable { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + + /** + * Rocksdb ConsumeQueue's store unit. Format: + * + * 
+     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬───────────────────────┐
+     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  QueueId  │  CTRL_1   │  ConsumeQueue Offset  │
+     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │     (8 Bytes)         │
+     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴───────────────────────┤
+     * │                                                    Key Unit                                                             │
+     * │                                                                                                                         │
+     *
+ * + * 
+     * ┌─────────────────────────────┬───────────────────┬──────────────────┬──────────────────┐
+     * │  CommitLog Physical Offset  │      Body Size    │   Tag HashCode   │  Msg Store Time  │
+     * │        (8 Bytes)            │      (4 Bytes)    │    (8 Bytes)     │    (8 Bytes)     │
+     * ├─────────────────────────────┴───────────────────┴──────────────────┴──────────────────┤
+     * │                                                    Value Unit                         │
+     * │                                                                                       │
+     *
+ * ConsumeQueue's store unit. Size: + * CommitLog Physical Offset(8) + Body Size(4) + Tag HashCode(8) + Msg Store Time(8) = 28 Bytes + */ + private static final int PHY_OFFSET_OFFSET = 0; + private static final int PHY_MSG_LEN_OFFSET = 8; + private static final int MSG_TAG_HASHCODE_OFFSET = 12; + private static final int MSG_STORE_TIME_SIZE_OFFSET = 20; + public static final int CQ_UNIT_SIZE = 8 + 4 + 8 + 8; + + /** + * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬───────────────────────┐ + * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_1 │ ConsumeQueue Offset │ + * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ (8 Bytes) │ + * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴───────────────────────┤ + */ + private static final int CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1 + 8; + + /** + * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────────────┐ + * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_0(CTRL_2) │ + * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ + * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────────────┤ + */ + private static final int DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1; + + private final ConsumeQueueRocksDBStorage rocksDBStorage; + private final DefaultMessageStore messageStore; + + private ColumnFamilyHandle defaultCFH; + + public RocksDBConsumeQueueTable(ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { + this.rocksDBStorage = rocksDBStorage; + this.messageStore = messageStore; + } + + public void load() { + this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle(); + } + + public void buildAndPutCQByteBuffer(final Pair cqBBPair, + final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException { + final ByteBuffer cqKey = cqBBPair.getObject1(); + buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset()); + + final ByteBuffer cqValue = cqBBPair.getObject2(); + buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp()); + + writeBatch.put(this.defaultCFH, cqKey, cqValue); + } + + public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset); + byte[] value = this.rocksDBStorage.getCQ(keyBB.array()); + return (value != null) ? ByteBuffer.wrap(value) : null; + } + + public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final List defaultCFHList = new ArrayList(num); + final ByteBuffer[] resultList = new ByteBuffer[num]; + final List kvIndexList = new ArrayList(num); + final List kvKeyList = new ArrayList(num); + for (int i = 0; i < num; i++) { + final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i); + kvIndexList.add(i); + kvKeyList.add(keyBB.array()); + defaultCFHList.add(this.defaultCFH); + } + int keyNum = kvIndexList.size(); + if (keyNum > 0) { + List kvValueList = this.rocksDBStorage.multiGet(defaultCFHList, kvKeyList); + final int valueNum = kvValueList.size(); + if (keyNum != valueNum) { + throw new RocksDBException("rocksdb bug, multiGet"); + } + for (int i = 0; i < valueNum; i++) { + byte[] value = kvValueList.get(i); + if (value == null) { + continue; + } + ByteBuffer byteBuffer = ByteBuffer.wrap(value); + resultList[kvIndexList.get(i)] = byteBuffer; + } + } + + final int resultSize = resultList.length; + List bbValueList = new ArrayList(resultSize); + for (int i = 0; i < resultSize; i++) { + ByteBuffer byteBuffer = resultList[i]; + if (byteBuffer == null) { + break; + } + bbValueList.add(byteBuffer); + } + return bbValueList; + } + + /** + * When topic is deleted, we clean up its CqUnit in rocksdb. + * @param topic + * @param queueId + * @throws RocksDBException + */ + public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException { + final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId); + final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId); + + writeBatch.deleteRange(this.defaultCFH, cqStartKey.array(), cqEndKey.array()); + + log.info("Rocksdb consumeQueue table delete topic. {}, {}", topic, queueId); + } + + public long binarySearchInCQByTime(String topic, int queueId, long high, long low, long timestamp, + long minPhysicOffset) throws RocksDBException { + long result = 0; + long targetOffset = -1L, leftOffset = -1L, rightOffset = -1L; + long leftValue = -1L, rightValue = -1L; + while (high >= low) { + long midOffset = low + ((high - low) >>> 1); + ByteBuffer byteBuffer = getCQInKV(topic, queueId, midOffset); + if (byteBuffer == null) { + ERROR_LOG.warn("binarySearchInCQByTimeStamp Failed. topic: {}, queueId: {}, timestamp: {}, result: null", + topic, queueId, timestamp); + low = midOffset + 1; + continue; + } + + long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET); + if (phyOffset < minPhysicOffset) { + low = midOffset + 1; + leftOffset = midOffset; + continue; + } + long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET); + if (storeTime < 0) { + return 0; + } else if (storeTime == timestamp) { + targetOffset = midOffset; + break; + } else if (storeTime > timestamp) { + high = midOffset - 1; + rightOffset = midOffset; + rightValue = storeTime; + } else { + low = midOffset + 1; + leftOffset = midOffset; + leftValue = storeTime; + } + } + if (targetOffset != -1) { + result = targetOffset; + } else { + if (leftValue == -1) { + result = rightOffset; + } else if (rightValue == -1) { + result = leftOffset; + } else { + result = Math.abs(timestamp - leftValue) > Math.abs(timestamp - rightValue) ? rightOffset : leftOffset; + } + } + return result; + } + + public PhyAndCQOffset binarySearchInCQ(String topic, int queueId, long high, long low, long targetPhyOffset, + boolean min) throws RocksDBException { + long resultCQOffset = -1L; + long resultPhyOffset = -1L; + while (high >= low) { + long midCQOffset = low + ((high - low) >>> 1); + ByteBuffer byteBuffer = getCQInKV(topic, queueId, midCQOffset); + if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { + ROCKSDB_LOG.warn("binarySearchInCQ. {}, {}, {}, {}, {}", topic, queueId, midCQOffset, low, high); + } + if (byteBuffer == null) { + low = midCQOffset + 1; + continue; + } + + final long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET); + if (phyOffset == targetPhyOffset) { + if (min) { + resultCQOffset = midCQOffset; + resultPhyOffset = phyOffset; + } + break; + } else if (phyOffset > targetPhyOffset) { + high = midCQOffset - 1; + if (min) { + resultCQOffset = midCQOffset; + resultPhyOffset = phyOffset; + } + } else { + low = midCQOffset + 1; + if (!min) { + resultCQOffset = midCQOffset; + resultPhyOffset = phyOffset; + } + } + } + return new PhyAndCQOffset(resultPhyOffset, resultCQOffset); + } + + public static Pair getCQByteBufferPair() { + ByteBuffer cqKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN); + ByteBuffer cqValue = ByteBuffer.allocateDirect(CQ_UNIT_SIZE); + return new Pair<>(cqKey, cqValue); + } + + private ByteBuffer buildCQKeyByteBuffer(final byte[] topicBytes, final int queueId, final long cqOffset) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset); + return byteBuffer; + } + + private void buildCQKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) { + byteBuffer.position(0).limit(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset); + } + + private void buildCQKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) { + byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(CTRL_1).putLong(cqOffset); + byteBuffer.flip(); + } + + private void buildCQValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, final long tagsCode, final long storeTimestamp) { + byteBuffer.position(0).limit(CQ_UNIT_SIZE); + buildCQValueByteBuffer0(byteBuffer, phyOffset, msgSize, tagsCode, storeTimestamp); + } + + private void buildCQValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, + final long tagsCode, final long storeTimestamp) { + byteBuffer.putLong(phyOffset).putInt(msgSize).putLong(tagsCode).putLong(storeTimestamp); + byteBuffer.flip(); + } + + private ByteBuffer buildDeleteCQKey(final boolean start, final byte[] topicBytes, final int queueId) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); + + byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(start ? CTRL_0 : CTRL_2); + byteBuffer.flip(); + return byteBuffer; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java new file mode 100644 index 00000000000..aa796c4d39e --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.rocksdb; + +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.AbstractCompactionFilter; +import org.rocksdb.AbstractCompactionFilterFactory; +import org.rocksdb.RemoveConsumeQueueCompactionFilter; + +public class ConsumeQueueCompactionFilterFactory extends AbstractCompactionFilterFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + private final MessageStore messageStore; + + public ConsumeQueueCompactionFilterFactory(final MessageStore messageStore) { + this.messageStore = messageStore; + } + + @Override + public String name() { + return "ConsumeQueueCompactionFilterFactory"; + } + + @Override + public RemoveConsumeQueueCompactionFilter createCompactionFilter(final AbstractCompactionFilter.Context context) { + long minPhyOffset = this.messageStore.getMinPhyOffset(); + LOGGER.info("manualCompaction minPhyOffset: {}, isFull: {}, isManual: {}", + minPhyOffset, context.isFullCompaction(), context.isManualCompaction()); + return new RemoveConsumeQueueCompactionFilter(minPhyOffset); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java new file mode 100644 index 00000000000..362684560c8 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.rocksdb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.config.AbstractRocksDBStorage; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.CompactRangeOptions; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; + +public class ConsumeQueueRocksDBStorage extends AbstractRocksDBStorage { + private final MessageStore messageStore; + private volatile ColumnFamilyHandle offsetCFHandle; + + public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath, final int prefixLen) { + this.messageStore = messageStore; + this.dbPath = dbPath; + this.readOnly = false; + } + + private void initOptions() { + this.options = RocksDBOptionsFactory.createDBOptions(); + + this.writeOptions = new WriteOptions(); + this.writeOptions.setSync(false); + this.writeOptions.setDisableWAL(true); + this.writeOptions.setNoSlowdown(true); + + this.totalOrderReadOptions = new ReadOptions(); + this.totalOrderReadOptions.setPrefixSameAsStart(false); + this.totalOrderReadOptions.setTotalOrderSeek(false); + + this.compactRangeOptions = new CompactRangeOptions(); + this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce); + this.compactRangeOptions.setAllowWriteStall(true); + this.compactRangeOptions.setExclusiveManualCompaction(false); + this.compactRangeOptions.setChangeLevel(true); + this.compactRangeOptions.setTargetLevel(-1); + this.compactRangeOptions.setMaxSubcompactions(4); + } + + @Override + protected boolean postLoad() { + try { + UtilAll.ensureDirOK(this.dbPath); + + initOptions(); + + final List cfDescriptors = new ArrayList(); + + ColumnFamilyOptions cqCfOptions = RocksDBOptionsFactory.createCQCFOptions(this.messageStore); + this.cfOptions.add(cqCfOptions); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cqCfOptions)); + + ColumnFamilyOptions offsetCfOptions = RocksDBOptionsFactory.createOffsetCFOptions(); + this.cfOptions.add(offsetCfOptions); + cfDescriptors.add(new ColumnFamilyDescriptor("offset".getBytes(DataConverter.CHARSET_UTF8), offsetCfOptions)); + + final List cfHandles = new ArrayList(); + open(cfDescriptors, cfHandles); + + this.defaultCFHandle = cfHandles.get(0); + this.offsetCFHandle = cfHandles.get(1); + } catch (final Exception e) { + LOGGER.error("postLoad Failed. {}", this.dbPath, e); + return false; + } + return true; + } + + @Override + protected void preShutdown() { + this.offsetCFHandle.close(); + } + + public byte[] getCQ(final byte[] keyBytes) throws RocksDBException { + return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes); + } + + public byte[] getOffset(final byte[] keyBytes) throws RocksDBException { + return get(this.offsetCFHandle, this.totalOrderReadOptions, keyBytes); + } + + public List multiGet(final List cfhList, final List keys) throws RocksDBException { + return multiGet(this.totalOrderReadOptions, cfhList, keys); + } + + public void batchPut(final WriteBatch batch) throws RocksDBException { + batchPut(this.writeOptions, batch); + } + + public void manualCompaction(final long minPhyOffset) { + try { + manualCompaction(minPhyOffset, this.compactRangeOptions); + } catch (Exception e) { + LOGGER.error("manualCompaction Failed. minPhyOffset: {}", minPhyOffset, e); + } + } + + public RocksIterator seekOffsetCF() { + return this.db.newIterator(this.offsetCFHandle, this.totalOrderReadOptions); + } + + public ColumnFamilyHandle getOffsetCFHandle() { + return this.offsetCFHandle; + } +} \ No newline at end of file diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java new file mode 100644 index 00000000000..a3a99d3346c --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.rocksdb; + +import org.apache.rocketmq.common.config.ConfigRocksDBStorage; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.CompactionOptionsUniversal; +import org.rocksdb.CompactionStopStyle; +import org.rocksdb.CompactionStyle; +import org.rocksdb.CompressionType; +import org.rocksdb.DBOptions; +import org.rocksdb.DataBlockIndexType; +import org.rocksdb.IndexType; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.LRUCache; +import org.rocksdb.RateLimiter; +import org.rocksdb.SkipListMemTableConfig; +import org.rocksdb.Statistics; +import org.rocksdb.StatsLevel; +import org.rocksdb.StringAppendOperator; +import org.rocksdb.WALRecoveryMode; +import org.rocksdb.util.SizeUnit; + +public class RocksDBOptionsFactory { + + public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageStore) { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). + setFormatVersion(5). + setIndexType(IndexType.kBinarySearch). + setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash). + setDataBlockHashTableUtilRatio(0.75). + setBlockSize(32 * SizeUnit.KB). + setMetadataBlockSize(4 * SizeUnit.KB). + setFilterPolicy(new BloomFilter(16, false)). + setCacheIndexAndFilterBlocks(false). + setCacheIndexAndFilterBlocksWithHighPriority(true). + setPinL0FilterAndIndexBlocksInCache(false). + setPinTopLevelIndexAndFilter(true). + setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false)). + setWholeKeyFiltering(true); + + ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); + CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal(); + compactionOption.setSizeRatio(100). + setMaxSizeAmplificationPercent(25). + setAllowTrivialMove(true). + setMinMergeWidth(2). + setMaxMergeWidth(Integer.MAX_VALUE). + setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize). + setCompressionSizePercent(-1); + return columnFamilyOptions.setMaxWriteBufferNumber(4). + setWriteBufferSize(128 * SizeUnit.MB). + setMinWriteBufferNumberToMerge(1). + setTableFormatConfig(blockBasedTableConfig). + setMemTableConfig(new SkipListMemTableConfig()). + setCompressionType(CompressionType.LZ4_COMPRESSION). + setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION). + setNumLevels(7). + setCompactionStyle(CompactionStyle.UNIVERSAL). + setCompactionOptionsUniversal(compactionOption). + setMaxCompactionBytes(100 * SizeUnit.GB). + setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB). + setHardPendingCompactionBytesLimit(256 * SizeUnit.GB). + setLevel0FileNumCompactionTrigger(2). + setLevel0SlowdownWritesTrigger(8). + setLevel0StopWritesTrigger(10). + setTargetFileSizeBase(256 * SizeUnit.MB). + setTargetFileSizeMultiplier(2). + setMergeOperator(new StringAppendOperator()). + setCompactionFilterFactory(new ConsumeQueueCompactionFilterFactory(messageStore)). + setReportBgIoStats(true). + setOptimizeFiltersForHits(true); + } + + public static ColumnFamilyOptions createOffsetCFOptions() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). + setFormatVersion(5). + setIndexType(IndexType.kBinarySearch). + setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch). + setBlockSize(32 * SizeUnit.KB). + setFilterPolicy(new BloomFilter(16, false)). + setCacheIndexAndFilterBlocks(false). + setCacheIndexAndFilterBlocksWithHighPriority(true). + setPinL0FilterAndIndexBlocksInCache(false). + setPinTopLevelIndexAndFilter(true). + setBlockCache(new LRUCache(128 * SizeUnit.MB, 8, false)). + setWholeKeyFiltering(true); + + ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); + return columnFamilyOptions.setMaxWriteBufferNumber(4). + setWriteBufferSize(64 * SizeUnit.MB). + setMinWriteBufferNumberToMerge(1). + setTableFormatConfig(blockBasedTableConfig). + setMemTableConfig(new SkipListMemTableConfig()). + setCompressionType(CompressionType.NO_COMPRESSION). + setNumLevels(7). + setCompactionStyle(CompactionStyle.LEVEL). + setLevel0FileNumCompactionTrigger(2). + setLevel0SlowdownWritesTrigger(8). + setLevel0StopWritesTrigger(10). + setTargetFileSizeBase(64 * SizeUnit.MB). + setTargetFileSizeMultiplier(2). + setMaxBytesForLevelBase(256 * SizeUnit.MB). + setMaxBytesForLevelMultiplier(2). + setMergeOperator(new StringAppendOperator()). + setInplaceUpdateSupport(true); + } + + /** + * Create a rocksdb db options, the user must take care to close it after closing db. + * @return + */ + public static DBOptions createDBOptions() { + //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide + // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java + DBOptions options = new DBOptions(); + Statistics statistics = new Statistics(); + statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS); + return options. + setDbLogDir(ConfigRocksDBStorage.getDBLogDir()). + setInfoLogLevel(InfoLogLevel.INFO_LEVEL). + setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery). + setManualWalFlush(true). + setMaxTotalWalSize(0). + setWalSizeLimitMB(0). + setWalTtlSeconds(0). + setCreateIfMissing(true). + setCreateMissingColumnFamilies(true). + setMaxOpenFiles(-1). + setMaxLogFileSize(1 * SizeUnit.GB). + setKeepLogFileNum(5). + setMaxManifestFileSize(1 * SizeUnit.GB). + setAllowConcurrentMemtableWrite(false). + setStatistics(statistics). + setAtomicFlush(true). + setMaxBackgroundJobs(32). + setMaxSubcompactions(8). + setParanoidChecks(true). + setDelayedWriteRate(16 * SizeUnit.MB). + setRateLimiter(new RateLimiter(100 * SizeUnit.MB)). + setUseDirectIoForFlushAndCompaction(false). + setUseDirectReads(false); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java index 2dd3fc5b52a..489d7b4fbce 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.store.stats; import java.util.HashMap; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import org.apache.commons.lang3.tuple.Pair; import org.apache.rocketmq.common.BrokerConfig; @@ -32,13 +31,14 @@ import org.apache.rocketmq.common.statistics.StatisticsItemStateGetter; import org.apache.rocketmq.common.statistics.StatisticsKindMeta; import org.apache.rocketmq.common.statistics.StatisticsManager; +import org.apache.rocketmq.common.stats.MomentStatsItemSet; import org.apache.rocketmq.common.stats.Stats; +import org.apache.rocketmq.common.stats.StatsItem; +import org.apache.rocketmq.common.stats.StatsItemSet; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; -import org.apache.rocketmq.common.stats.MomentStatsItemSet; -import org.apache.rocketmq.common.stats.StatsItem; -import org.apache.rocketmq.common.stats.StatsItemSet; public class BrokerStatsManager { @@ -281,11 +281,11 @@ public boolean online(StatisticsItem item) { private void initScheduleService() { this.scheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig)); + ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig)); this.commercialExecutor = - Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig)); + ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig)); this.accountExecutor = - Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig)); + ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig)); } public MomentStatsItemSet getMomentStatsItemSetFallSize() { diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index 690f4863e61..3ab51a26d38 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -35,13 +35,13 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.TopicFilterType; @@ -54,9 +54,9 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; -import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; @@ -65,6 +65,9 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.CqUnit; +import org.apache.rocketmq.store.queue.ReferredIterator; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.util.PerfCounter; @@ -174,11 +177,11 @@ public TimerMessageStore(final MessageStore messageStore, final MessageStoreConf this.lastBrokerRole = storeConfig.getBrokerRole(); if (messageStore instanceof DefaultMessageStore) { - scheduler = Executors.newSingleThreadScheduledExecutor( + scheduler = ThreadUtils.newSingleThreadScheduledExecutor( new ThreadFactoryImpl("TimerScheduledThread", ((DefaultMessageStore) messageStore).getBrokerIdentity())); } else { - scheduler = Executors.newSingleThreadScheduledExecutor( + scheduler = ThreadUtils.newSingleThreadScheduledExecutor( new ThreadFactoryImpl("TimerScheduledThread")); } @@ -222,7 +225,7 @@ public void initService() { dequeueGetMessageServices[i] = new TimerDequeueGetMessageService(); } - int putThreadNum = Math.max(storeConfig.getTimerGetMessageThreadNum(), 1); + int putThreadNum = Math.max(storeConfig.getTimerPutMessageThreadNum(), 1); dequeuePutMessageServices = new TimerDequeuePutMessageService[putThreadNum]; for (int i = 0; i < dequeuePutMessageServices.length; i++) { dequeuePutMessageServices[i] = new TimerDequeuePutMessageService(); @@ -332,7 +335,7 @@ public long reviseQueueOffset(long processOffset) { // if not, use cq offset. long msgQueueOffset = messageExt.getQueueOffset(); int queueId = messageExt.getQueueId(); - ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); if (null == cq) { return msgQueueOffset; } @@ -345,15 +348,18 @@ public long reviseQueueOffset(long processOffset) { offsetPy, sizePy); break; } - SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(tmpOffset); - if (null == bufferCQ) { - // offset in msg may be greater than offset of cq. - tmpOffset -= 1; - continue; - } + ReferredIterator iterator = null; try { - long offsetPyTemp = bufferCQ.getByteBuffer().getLong(); - int sizePyTemp = bufferCQ.getByteBuffer().getInt(); + iterator = cq.iterateFrom(tmpOffset); + CqUnit cqUnit = null; + if (null == iterator || (cqUnit = iterator.next()) == null) { + // offset in msg may be greater than offset of cq. + tmpOffset -= 1; + continue; + } + + long offsetPyTemp = cqUnit.getPos(); + int sizePyTemp = cqUnit.getSize(); if (offsetPyTemp == offsetPy && sizePyTemp == sizePy) { LOGGER.info("reviseQueueOffset check cq offset ok. {}, {}, {}", tmpOffset, offsetPyTemp, sizePyTemp); @@ -364,7 +370,9 @@ public long reviseQueueOffset(long processOffset) { } catch (Throwable e) { LOGGER.error("reviseQueueOffset check cq offset error.", e); } finally { - bufferCQ.release(); + if (iterator != null) { + iterator.release(); + } } } @@ -599,7 +607,12 @@ public void addMetric(MessageExt msg, int value) { if (null == msg || null == msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC)) { return; } - timerMetrics.addAndGet(msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC), value); + if (msg.getProperty(TIMER_ENQUEUE_MS) != null + && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) { + return; + } + // pass msg into addAndGet, for further more judgement extension. + timerMetrics.addAndGet(msg, value); } catch (Throwable t) { if (frequency.incrementAndGet() % 1000 == 0) { LOGGER.error("error in adding metric", t); @@ -627,7 +640,7 @@ public boolean enqueue(int queueId) { if (!isRunningEnqueue()) { return false; } - ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); if (null == cq) { return false; } @@ -637,18 +650,22 @@ public boolean enqueue(int queueId) { currQueueOffset = cq.getMinOffsetInQueue(); } long offset = currQueueOffset; - SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(offset); - if (null == bufferCQ) { - return false; - } + ReferredIterator iterator = null; try { + iterator = cq.iterateFrom(offset); + if (null == iterator) { + return false; + } + int i = 0; - for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { + while (iterator.hasNext()) { + i++; perfCounterTicks.startTick("enqueue_get"); try { - long offsetPy = bufferCQ.getByteBuffer().getLong(); - int sizePy = bufferCQ.getByteBuffer().getInt(); - bufferCQ.getByteBuffer().getLong(); //tags code + CqUnit cqUnit = iterator.next(); + long offsetPy = cqUnit.getPos(); + int sizePy = cqUnit.getSize(); + cqUnit.getTagsCode(); //tags code MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy); if (null == msgExt) { perfCounterTicks.getCounter("enqueue_get_miss"); @@ -657,7 +674,7 @@ public boolean enqueue(int queueId) { lastEnqueueButExpiredStoreTime = msgExt.getStoreTimestamp(); long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); // use CQ offset, not offset in Message - msgExt.setQueueOffset(offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE)); + msgExt.setQueueOffset(offset + i); TimerRequest timerRequest = new TimerRequest(offsetPy, sizePy, delayedTime, System.currentTimeMillis(), MAGIC_DEFAULT, msgExt); // System.out.printf("build enqueue request, %s%n", timerRequest); while (!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) { @@ -681,14 +698,16 @@ public boolean enqueue(int queueId) { if (!isRunningEnqueue()) { return false; } - currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); + currQueueOffset = offset + i; } - currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); + currQueueOffset = offset + i; return i > 0; } catch (Exception e) { LOGGER.error("Unknown exception in enqueuing", e); } finally { - bufferCQ.release(); + if (iterator != null) { + iterator.release(); + } } return false; } @@ -1323,6 +1342,7 @@ protected void putMessageToTimerWheel(TimerRequest req) { perfCounterTicks.startTick(ENQUEUE_PUT); DefaultStoreMetricsManager.incTimerEnqueueCount(getRealTopic(req.getMsg())); if (shouldRunningDequeue && req.getDelayTime() < currWriteTimeMs) { + req.setEnqueueTime(Long.MAX_VALUE); dequeuePutQueue.put(req); } else { boolean doEnqueueRes = doEnqueue( @@ -1452,9 +1472,14 @@ public void run() { } try { perfCounterTicks.startTick(DEQUEUE_PUT); - DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(tr.getMsg())); - addMetric(tr.getMsg(), -1); - MessageExtBrokerInner msg = convert(tr.getMsg(), tr.getEnqueueTime(), needRoll(tr.getMagic())); + MessageExt msgExt = tr.getMsg(); + DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(msgExt)); + if (tr.getEnqueueTime() == Long.MAX_VALUE) { + // never enqueue, mark it. + MessageAccessor.putProperty(msgExt, TIMER_ENQUEUE_MS, String.valueOf(Long.MAX_VALUE)); + } + addMetric(msgExt, -1); + MessageExtBrokerInner msg = convert(msgExt, tr.getEnqueueTime(), needRoll(tr.getMagic())); doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic())); while (!doRes && !isStopped()) { if (!isRunningDequeue()) { @@ -1630,7 +1655,7 @@ public void run() { if (System.currentTimeMillis() - start > storeConfig.getTimerProgressLogIntervalMs()) { start = System.currentTimeMillis(); long tmpQueueOffset = currQueueOffset; - ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0); + ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); TimerMessageStore.LOGGER.info("[{}]Timer progress-check commitRead:[{}] currRead:[{}] currWrite:[{}] readBehind:{} currReadOffset:{} offsetBehind:{} behindMaster:{} " + "enqPutQueue:{} deqGetQueue:{} deqPutQueue:{} allCongestNum:{} enqExpiredStoreTime:{}", @@ -1673,7 +1698,7 @@ public boolean isReject(long deliverTimeMs) { public long getEnqueueBehindMessages() { long tmpQueueOffset = currQueueOffset; - ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0); + ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); return maxOffsetInQueue - tmpQueueOffset; } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java index e7b00cc073c..7f8fedd8a5b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java @@ -38,6 +38,8 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -78,7 +80,8 @@ public long updateDistPair(int period, int value) { return distPair.getCount().addAndGet(value); } - public long addAndGet(String topic, int value) { + public long addAndGet(MessageExt msg, int value) { + String topic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC); Metric pair = getTopicPair(topic); getDataVersion().nextVersion(); pair.setTimeStamp(System.currentTimeMillis()); diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java index 1dd64f75921..1b25d355c65 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java @@ -27,8 +27,9 @@ public class TimerRequest { private final int sizePy; private final long delayTime; - private final long enqueueTime; private final int magic; + + private long enqueueTime; private MessageExt msg; @@ -94,7 +95,9 @@ public void setDeleteList(Set deleteList) { public void setLatch(CountDownLatch latch) { this.latch = latch; } - + public void setEnqueueTime(long enqueueTime) { + this.enqueueTime = enqueueTime; + } public void idempotentRelease() { idempotentRelease(true); } diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java index 12d1e5723c8..1d09ca86ecb 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java @@ -428,9 +428,10 @@ private String buildMessageBodyByOffset(String message, long i) { private long getStoreTime(CqUnit cqUnit) { try { - Method getStoreTime = getDefaultMessageStore().getClass().getDeclaredMethod("getStoreTime", CqUnit.class); + Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass(); + Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class); getStoreTime.setAccessible(true); - return (long) getStoreTime.invoke(getDefaultMessageStore(), cqUnit); + return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { throw new RuntimeException(e); } diff --git a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java index 85626a332e9..2447bbf68f3 100644 --- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java @@ -28,9 +28,11 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.queue.MultiDispatch; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.rocksdb.RocksDBException; import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue; import static org.junit.Assert.assertEquals; @@ -69,15 +71,15 @@ public void destroy() { } @Test - public void queueKey() { + public void lmqQueueKey() { MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class); when(messageExtBrokerInner.getQueueId()).thenReturn(2); - String ret = consumeQueue.queueKey("%LMQ%lmq123", messageExtBrokerInner); + String ret = MultiDispatch.lmqQueueKey("%LMQ%lmq123"); assertEquals(ret, "%LMQ%lmq123-0"); } @Test - public void wrapMultiDispatch() { + public void wrapMultiDispatch() throws RocksDBException { MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue(); messageStore.assignOffset(messageExtBrokerInner); assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0"); diff --git a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java new file mode 100644 index 00000000000..acf5edf5117 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java @@ -0,0 +1,1060 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store; + +import java.io.File; +import java.io.RandomAccessFile; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.collect.Sets; + +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.MessageBatch; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBatch; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.config.FlushDiskType; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.CqUnit; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.assertj.core.util.Strings; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class RocksDBMessageStoreTest { + private final String storeMessage = "Once, there was a chance for me!"; + private final String messageTopic = "FooBar"; + private final String storeType = StoreType.DEFAULT_ROCKSDB.getStoreType(); + private int queueTotal = 100; + private AtomicInteger queueId = new AtomicInteger(0); + private SocketAddress bornHost; + private SocketAddress storeHost; + private byte[] messageBody; + private MessageStore messageStore; + + @Before + public void init() throws Exception { + if (notExecuted()) { + return; + } + storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + + messageStore = buildMessageStore(); + boolean load = messageStore.load(); + assertTrue(load); + messageStore.start(); + } + + @Test(expected = OverlappingFileLockException.class) + public void test_repeat_restart() throws Exception { + if (notExecuted()) { + throw new OverlappingFileLockException(); + } + queueTotal = 1; + messageBody = storeMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8); + messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4); + messageStoreConfig.setMaxHashSlotNum(100); + messageStoreConfig.setMaxIndexNum(100 * 10); + messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "store"); + messageStoreConfig.setHaListenPort(0); + MessageStore master = new RocksDBMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); + + boolean load = master.load(); + assertTrue(load); + + try { + master.start(); + master.start(); + } finally { + master.shutdown(); + master.destroy(); + } + } + + @After + public void destroy() { + if (notExecuted()) { + return; + } + messageStore.shutdown(); + messageStore.destroy(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + File file = new File(messageStoreConfig.getStorePathRootDir()); + UtilAll.deleteFile(file); + } + + private MessageStore buildMessageStore() throws Exception { + return buildMessageStore(null, ""); + } + + private MessageStore buildMessageStore(String storePathRootDir, String topic) throws Exception { + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10); + messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10); + messageStoreConfig.setMaxHashSlotNum(10000); + messageStoreConfig.setMaxIndexNum(100 * 100); + messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH); + messageStoreConfig.setFlushIntervalConsumeQueue(1); + messageStoreConfig.setStoreType(storeType); + messageStoreConfig.setHaListenPort(0); + if (Strings.isNullOrEmpty(storePathRootDir)) { + UUID uuid = UUID.randomUUID(); + storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-" + uuid.toString(); + } + messageStoreConfig.setStorePathRootDir(storePathRootDir); + ConcurrentMap topicConfigTable = new ConcurrentHashMap<>(); + topicConfigTable.put(topic, new TopicConfig(topic, 1, 1)); + return new RocksDBMessageStore(messageStoreConfig, + new BrokerStatsManager("simpleTest", true), + new MyMessageArrivingListener(), + new BrokerConfig(), topicConfigTable); + } + + @Test + public void testWriteAndRead() { + if (notExecuted()) { + return; + } + long ipv4HostMsgs = 10; + long ipv6HostMsgs = 10; + long totalMsgs = ipv4HostMsgs + ipv6HostMsgs; + queueTotal = 1; + messageBody = storeMessage.getBytes(); + for (long i = 0; i < ipv4HostMsgs; i++) { + messageStore.putMessage(buildMessage()); + } + + for (long i = 0; i < ipv6HostMsgs; i++) { + messageStore.putMessage(buildIPv6HostMessage()); + } + + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + for (long i = 0; i < totalMsgs; i++) { + GetMessageResult result = messageStore.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); + assertThat(result).isNotNull(); + result.release(); + } + verifyThatMasterIsFunctional(totalMsgs, messageStore); + } + + @Test + public void testLookMessageByOffset_OffsetIsFirst() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = new Random().nextInt(10); + String topic = "FooBar"; + int firstOffset = 0; + AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); + AppendMessageResult firstResult = appendMessageResultArray[0]; + + MessageExt messageExt = messageStore.lookMessageByOffset(firstResult.getWroteOffset()); + MessageExt messageExt1 = getDefaultMessageStore().lookMessageByOffset(firstResult.getWroteOffset(), firstResult.getWroteBytes()); + + assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset)); + assertThat(new String(messageExt1.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset)); + } + + @Test + public void testLookMessageByOffset_OffsetIsLast() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = new Random().nextInt(10); + String topic = "FooBar"; + AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); + int lastIndex = totalCount - 1; + AppendMessageResult lastResult = appendMessageResultArray[lastIndex]; + + MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastResult.getWroteOffset(), lastResult.getWroteBytes()); + + assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, lastIndex)); + } + + @Test + public void testLookMessageByOffset_OffsetIsOutOfBound() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = new Random().nextInt(10); + String topic = "FooBar"; + AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); + long lastOffset = getMaxOffset(appendMessageResultArray); + + MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastOffset); + + assertThat(messageExt).isNull(); + } + + @Test + public void testGetOffsetInQueueByTime() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); + for (AppendMessageResult appendMessageResult : appendMessageResults) { + long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp()); + CqUnit cqUnit = consumeQueue.get(offset); + assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset()); + assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes()); + } + } + + @Test + public void testGetOffsetInQueueByTime_TimestampIsSkewing() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + int skewing = 2; + + ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); + for (AppendMessageResult appendMessageResult : appendMessageResults) { + long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing); + CqUnit cqUnit = consumeQueue.get(offset); + assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset()); + assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes()); + } + } + + @Test + public void testGetOffsetInQueueByTime_TimestampSkewingIsLarge() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + int skewing = 20000; + + ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); + for (AppendMessageResult appendMessageResult : appendMessageResults) { + long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing); + CqUnit cqUnit = consumeQueue.get(offset); + assertThat(cqUnit.getPos()).isEqualTo(appendMessageResults[0].getWroteOffset()); + assertThat(cqUnit.getSize()).isEqualTo(appendMessageResults[0].getWroteBytes()); + } + } + + @Test + public void testGetOffsetInQueueByTime_ConsumeQueueNotFound1() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + int wrongQueueId = 1; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); + //Thread.sleep(10); + + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + long offset = messageStore.getOffsetInQueueByTime(topic, wrongQueueId, appendMessageResults[0].getStoreTimestamp()); + + assertThat(offset).isEqualTo(0); + } + + @Test + public void testGetOffsetInQueueByTime_ConsumeQueueNotFound2() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + int wrongQueueId = 1; + String topic = "FooBar"; + putMessages(totalCount, topic, queueId, false); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, 0); + + assertThat(messageStoreTimeStamp).isEqualTo(-1); + } + + @Test + public void testGetOffsetInQueueByTime_ConsumeQueueOffsetNotExist() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + int wrongQueueId = 1; + String topic = "FooBar"; + putMessages(totalCount, topic, queueId, true); + //Thread.sleep(10); + + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, -1); + + assertThat(messageStoreTimeStamp).isEqualTo(-1); + } + + @Test + public void testGetMessageStoreTimeStamp() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); + int minOffsetInQueue = (int) consumeQueue.getMinOffsetInQueue(); + for (int i = minOffsetInQueue; i < consumeQueue.getMaxOffsetInQueue(); i++) { + long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, i); + assertThat(messageStoreTimeStamp).isEqualTo(appendMessageResults[i].getStoreTimestamp()); + } + } + + @Test + public void testGetStoreTime_ParamIsNull() { + if (notExecuted()) { + return; + } + long storeTime = getStoreTime(null); + + assertThat(storeTime).isEqualTo(-1); + } + + @Test + public void testGetStoreTime_EverythingIsOk() { + if (notExecuted()) { + return; + } + final int totalCount = 10; + int queueId = 0; + String topic = "FooBar"; + AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); + //Thread.sleep(10); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, queueId); + + for (int i = 0; i < totalCount; i++) { + CqUnit cqUnit = consumeQueue.get(i); + long storeTime = getStoreTime(cqUnit); + assertThat(storeTime).isEqualTo(appendMessageResults[i].getStoreTimestamp()); + } + } + + @Test + public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() { + if (notExecuted()) { + return; + } + long phyOffset = -10; + int size = 138; + CqUnit cqUnit = new CqUnit(0, phyOffset, size, 0); + long storeTime = getStoreTime(cqUnit); + + assertThat(storeTime).isEqualTo(-1); + } + + @Test + public void testPutMessage_whenMessagePropertyIsTooLong() { + if (notExecuted()) { + return; + } + String topicName = "messagePropertyIsTooLongTest"; + MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1); + assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); + assertEquals(0L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue()); + MessageExtBrokerInner normalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, 100); + assertEquals(messageStore.putMessage(normalMessage).getPutMessageStatus(), PutMessageStatus.PUT_OK); + assertEquals(1L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue()); + } + + private RocksDBMessageStore getDefaultMessageStore() { + return (RocksDBMessageStore) this.messageStore; + } + + private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId) { + return putMessages(totalCount, topic, queueId, false); + } + + private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId, boolean interval) { + AppendMessageResult[] appendMessageResultArray = new AppendMessageResult[totalCount]; + for (int i = 0; i < totalCount; i++) { + String messageBody = buildMessageBodyByOffset(storeMessage, i); + + MessageExtBrokerInner msgInner = + i < totalCount / 2 ? buildMessage(messageBody.getBytes(), topic) : buildIPv6HostMessage(messageBody.getBytes(), topic); + msgInner.setQueueId(queueId); + PutMessageResult result = messageStore.putMessage(msgInner); + appendMessageResultArray[i] = result.getAppendMessageResult(); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); + if (interval) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException("Thread sleep ERROR"); + } + } + } + return appendMessageResultArray; + } + + private long getMaxOffset(AppendMessageResult[] appendMessageResultArray) { + if (appendMessageResultArray == null) { + return 0; + } + AppendMessageResult last = appendMessageResultArray[appendMessageResultArray.length - 1]; + return last.getWroteOffset() + last.getWroteBytes(); + } + + private String buildMessageBodyByOffset(String message, long i) { + return String.format("%s offset %d", message, i); + } + + private long getStoreTime(CqUnit cqUnit) { + try { + Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass(); + Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class); + getStoreTime.setAccessible(true); + return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + + private MessageExtBrokerInner buildMessage(byte[] messageBody, String topic) { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(messageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); + msg.setSysFlag(0); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(storeHost); + msg.setBornHost(bornHost); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + return msg; + } + + private MessageExtBrokerInner buildSpecifyLengthPropertyMessage(byte[] messageBody, String topic, int length) { + StringBuilder stringBuilder = new StringBuilder(); + Random random = new Random(); + for (int i = 0; i < length; i++) { + stringBuilder.append(random.nextInt(10)); + } + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.putUserProperty("test", stringBuilder.toString()); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(messageBody); + msg.setQueueId(0); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(storeHost); + msg.setBornHost(bornHost); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + return msg; + } + + private MessageExtBrokerInner buildIPv6HostMessage(byte[] messageBody, String topic) { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(messageBody); + msg.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); + msg.setSysFlag(0); + msg.setBornHostV6Flag(); + msg.setStoreHostAddressV6Flag(); + msg.setBornTimestamp(System.currentTimeMillis()); + try { + msg.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + try { + msg.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + return msg; + } + + private MessageExtBrokerInner buildMessage() { + return buildMessage(messageBody, messageTopic); + } + + public MessageExtBatch buildMessageBatch(MessageBatch msgBatch) { + MessageExtBatch msgExtBatch = new MessageExtBatch(); + msgExtBatch.setTopic(messageTopic); + msgExtBatch.setTags("TAG1"); + msgExtBatch.setKeys("Hello"); + msgExtBatch.setBody(msgBatch.getBody()); + msgExtBatch.setKeys(String.valueOf(System.currentTimeMillis())); + msgExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); + msgExtBatch.setSysFlag(0); + msgExtBatch.setBornTimestamp(System.currentTimeMillis()); + msgExtBatch.setStoreHost(storeHost); + msgExtBatch.setBornHost(bornHost); + return msgExtBatch; + } + + @Test + public void testGroupCommit() throws Exception { + if (notExecuted()) { + return; + } + long totalMsgs = 10; + queueTotal = 1; + messageBody = storeMessage.getBytes(); + for (long i = 0; i < totalMsgs; i++) { + messageStore.putMessage(buildMessage()); + } + + for (long i = 0; i < totalMsgs; i++) { + GetMessageResult result = messageStore.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + assertThat(result).isNotNull(); + result.release(); + } + verifyThatMasterIsFunctional(totalMsgs, messageStore); + } + + @Test + public void testMaxOffset() throws InterruptedException { + if (notExecuted()) { + return; + } + int firstBatchMessages = 3; + int queueId = 0; + messageBody = storeMessage.getBytes(); + + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(0); + + for (int i = 0; i < firstBatchMessages; i++) { + final MessageExtBrokerInner msg = buildMessage(); + msg.setQueueId(queueId); + messageStore.putMessage(msg); + } + + while (messageStore.dispatchBehindBytes() != 0) { + TimeUnit.MILLISECONDS.sleep(1); + } + + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages); + + // Disable the dispatcher + messageStore.getDispatcherList().clear(); + + int secondBatchMessages = 2; + + for (int i = 0; i < secondBatchMessages; i++) { + final MessageExtBrokerInner msg = buildMessage(); + msg.setQueueId(queueId); + messageStore.putMessage(msg); + } + + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages); + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, true)).isEqualTo(firstBatchMessages); + assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, false)).isEqualTo(firstBatchMessages + secondBatchMessages); + } + + private MessageExtBrokerInner buildIPv6HostMessage() { + return buildIPv6HostMessage(messageBody, "FooBar"); + } + + private void verifyThatMasterIsFunctional(long totalMsgs, MessageStore master) { + for (long i = 0; i < totalMsgs; i++) { + master.putMessage(buildMessage()); + } + + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + for (long i = 0; i < totalMsgs; i++) { + GetMessageResult result = master.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); + assertThat(result).isNotNull(); + result.release(); + + } + } + + @Test + public void testPullSize() throws Exception { + if (notExecuted()) { + return; + } + String topic = "pullSizeTopic"; + + for (int i = 0; i < 32; i++) { + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + // wait for consume queue build + // the sleep time should be great than consume queue flush interval + //Thread.sleep(100); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + String group = "simple"; + GetMessageResult getMessageResult32 = messageStore.getMessage(group, topic, 0, 0, 32, null); + assertThat(getMessageResult32.getMessageBufferList().size()).isEqualTo(32); + getMessageResult32.release(); + + GetMessageResult getMessageResult20 = messageStore.getMessage(group, topic, 0, 0, 20, null); + assertThat(getMessageResult20.getMessageBufferList().size()).isEqualTo(20); + + getMessageResult20.release(); + GetMessageResult getMessageResult45 = messageStore.getMessage(group, topic, 0, 0, 10, null); + assertThat(getMessageResult45.getMessageBufferList().size()).isEqualTo(10); + getMessageResult45.release(); + + } + + @Test + public void testRecover() throws Exception { + if (notExecuted()) { + return; + } + String topic = "recoverTopic"; + messageBody = storeMessage.getBytes(); + for (int i = 0; i < 100; i++) { + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + + // Thread.sleep(100);//wait for build consumer queue + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + + long maxPhyOffset = messageStore.getMaxPhyOffset(); + long maxCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); + + //1.just reboot + messageStore.shutdown(); + String storeRootDir = ((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir(); + messageStore = buildMessageStore(storeRootDir, topic); + boolean load = messageStore.load(); + assertTrue(load); + messageStore.start(); + assertTrue(maxPhyOffset == messageStore.getMaxPhyOffset()); + assertTrue(maxCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); + + //2.damage commit-log and reboot normal + for (int i = 0; i < 100; i++) { + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + //Thread.sleep(100); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + long secondLastPhyOffset = messageStore.getMaxPhyOffset(); + long secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); + + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + + + messageStore.shutdown(); + + //damage last message + damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset); + + //reboot + messageStore = buildMessageStore(storeRootDir, topic); + load = messageStore.load(); + assertTrue(load); + messageStore.start(); + assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset()); + assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); + + //3.damage commitlog and reboot abnormal + for (int i = 0; i < 100; i++) { + messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + //Thread.sleep(100); + StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); + secondLastPhyOffset = messageStore.getMaxPhyOffset(); + secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); + + messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + messageStore.shutdown(); + + //damage last message + damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset); + //add abort file + String fileName = StorePathConfigHelper.getAbortFile(((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir()); + File file = new File(fileName); + UtilAll.ensureDirOK(file.getParent()); + file.createNewFile(); + + messageStore = buildMessageStore(storeRootDir, topic); + load = messageStore.load(); + assertTrue(load); + messageStore.start(); + assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset()); + assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); + + //message write again + for (int i = 0; i < 100; i++) { + messageExtBrokerInner = buildMessage(); + messageExtBrokerInner.setTopic(topic); + messageExtBrokerInner.setQueueId(0); + messageStore.putMessage(messageExtBrokerInner); + } + } + + @Test + public void testStorePathOK() { + if (notExecuted()) { + return; + } + if (messageStore instanceof RocksDBMessageStore) { + assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathPhysic())); + assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathLogic())); + } + } + + private boolean fileExists(String path) { + if (path != null) { + File f = new File(path); + return f.exists(); + } + return false; + } + + private void damageCommitLog(RocksDBMessageStore store, long offset) throws Exception { + assertThat(store).isNotNull(); + MessageStoreConfig messageStoreConfig = store.getMessageStoreConfig(); + File file = new File(messageStoreConfig.getStorePathCommitLog() + File.separator + "00000000000000000000"); + try (RandomAccessFile raf = new RandomAccessFile(file, "rw"); + FileChannel fileChannel = raf.getChannel()) { + MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, 1024 * 1024 * 10); + int bodyLen = mappedByteBuffer.getInt((int) offset + 84); + int topicLenIndex = (int) offset + 84 + bodyLen + 4; + mappedByteBuffer.position(topicLenIndex); + mappedByteBuffer.putInt(0); + mappedByteBuffer.putInt(0); + mappedByteBuffer.putInt(0); + mappedByteBuffer.putInt(0); + mappedByteBuffer.force(); + fileChannel.force(true); + } + } + + @Test + public void testPutMsgExceedsMaxLength() { + if (notExecuted()) { + return; + } + messageBody = new byte[4 * 1024 * 1024 + 1]; + MessageExtBrokerInner msg = buildMessage(); + + PutMessageResult result = messageStore.putMessage(msg); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL); + } + + @Test + public void testPutMsgBatchExceedsMaxLength() { + if (notExecuted()) { + return; + } + messageBody = new byte[4 * 1024 * 1024 + 1]; + MessageExtBrokerInner msg1 = buildMessage(); + MessageExtBrokerInner msg2 = buildMessage(); + MessageExtBrokerInner msg3 = buildMessage(); + + MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3)); + msgBatch.setBody(msgBatch.encode()); + + MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch); + + try { + PutMessageResult result = this.messageStore.putMessages(msgExtBatch); + } catch (Exception e) { + assertThat(e.getMessage()).contains("message body size exceeded"); + } + } + + @Test + public void testPutMsgWhenReplicasNotEnough() { + if (notExecuted()) { + return; + } + MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig(); + messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER); + messageStoreConfig.setTotalReplicas(2); + messageStoreConfig.setInSyncReplicas(2); + messageStoreConfig.setEnableAutoInSyncReplicas(false); + ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true); + this.messageStore.setAliveReplicaNumInGroup(1); + + MessageExtBrokerInner msg = buildMessage(); + PutMessageResult result = this.messageStore.putMessage(msg); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH); + ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false); + } + + @Test + public void testPutMsgWhenAdaptiveDegradation() { + if (notExecuted()) { + return; + } + MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig(); + messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER); + messageStoreConfig.setTotalReplicas(2); + messageStoreConfig.setInSyncReplicas(2); + messageStoreConfig.setEnableAutoInSyncReplicas(true); + ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true); + this.messageStore.setAliveReplicaNumInGroup(1); + + MessageExtBrokerInner msg = buildMessage(); + PutMessageResult result = this.messageStore.putMessage(msg); + assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); + ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false); + messageStoreConfig.setEnableAutoInSyncReplicas(false); + } + + @Test + public void testGetBulkCommitLogData() { + if (notExecuted()) { + return; + } + RocksDBMessageStore defaultMessageStore = (RocksDBMessageStore) messageStore; + + messageBody = new byte[2 * 1024 * 1024]; + + for (int i = 0; i < 10; i++) { + MessageExtBrokerInner msg1 = buildMessage(); + messageStore.putMessage(msg1); + } + + List bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset()); + List msgList = new ArrayList<>(); + for (SelectMappedBufferResult bufferResult : bufferResultList) { + msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false)); + bufferResult.release(); + } + + assertThat(msgList.size()).isEqualTo(10); + } + + @Test + public void testPutLongMessage() throws Exception { + if (notExecuted()) { + return; + } + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + CommitLog commitLog = ((RocksDBMessageStore) messageStore).getCommitLog(); + MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig(); + MessageExtEncoder.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get(); + + //body size, topic size, properties size exactly equal to max size + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); + messageExtBrokerInner.setTopic(new String(new byte[127])); + messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE])); + PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult1 == null); + + //body size exactly more than max message body size + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]); + PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult2.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + + //body size exactly equal to max message size + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]); + PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult3.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + + //message properties length more than properties maxSize + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); + messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE + 1])); + PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult4.getPutMessageStatus() == PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); + + //message length more than buffer length capacity + messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); + messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE])); + messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE])); + PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); + assertTrue(encodeResult5.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + } + + @Test + public void testDynamicMaxMessageSize() { + if (notExecuted()) { + return; + } + MessageExtBrokerInner messageExtBrokerInner = buildMessage(); + MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig(); + int originMaxMessageSize = messageStoreConfig.getMaxMessageSize(); + + messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]); + PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner); + assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + + int newMaxMessageSize = originMaxMessageSize + 10; + messageStoreConfig.setMaxMessageSize(newMaxMessageSize); + putMessageResult = messageStore.putMessage(messageExtBrokerInner); + assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK); + + messageStoreConfig.setMaxMessageSize(10); + putMessageResult = messageStore.putMessage(messageExtBrokerInner); + assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); + + messageStoreConfig.setMaxMessageSize(originMaxMessageSize); + } + + @Test + public void testDeleteTopics() { + if (notExecuted()) { + return; + } + MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig(); + ConcurrentMap> consumeQueueTable = + ((RocksDBMessageStore) messageStore).getConsumeQueueTable(); + for (int i = 0; i < 10; i++) { + ConcurrentMap cqTable = new ConcurrentHashMap<>(); + String topicName = "topic-" + i; + for (int j = 0; j < 4; j++) { + ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(), + messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore); + cqTable.put(j, consumeQueue); + } + consumeQueueTable.put(topicName, cqTable); + } + Assert.assertEquals(consumeQueueTable.size(), 10); + HashSet resultSet = Sets.newHashSet("topic-3", "topic-5"); + messageStore.deleteTopics(Sets.difference(consumeQueueTable.keySet(), resultSet)); + Assert.assertEquals(consumeQueueTable.size(), 2); + Assert.assertEquals(resultSet, consumeQueueTable.keySet()); + } + + @Test + public void testCleanUnusedTopic() { + if (notExecuted()) { + return; + } + MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig(); + ConcurrentMap> consumeQueueTable = + ((RocksDBMessageStore) messageStore).getConsumeQueueTable(); + for (int i = 0; i < 10; i++) { + ConcurrentMap cqTable = new ConcurrentHashMap<>(); + String topicName = "topic-" + i; + for (int j = 0; j < 4; j++) { + ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(), + messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore); + cqTable.put(j, consumeQueue); + } + consumeQueueTable.put(topicName, cqTable); + } + Assert.assertEquals(consumeQueueTable.size(), 10); + HashSet resultSet = Sets.newHashSet("topic-3", "topic-5"); + messageStore.cleanUnusedTopic(resultSet); + Assert.assertEquals(consumeQueueTable.size(), 2); + Assert.assertEquals(resultSet, consumeQueueTable.keySet()); + } + + private class MyMessageArrivingListener implements MessageArrivingListener { + @Override + public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, + byte[] filterBitMap, Map properties) { + } + } + + private boolean notExecuted() { + return MixAll.isMac(); + } +} + + diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java index b2d99c3edba..17a2b5e19d7 100644 --- a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java +++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java @@ -38,11 +38,16 @@ public class StoreTestUtil { public static boolean isCommitLogAvailable(DefaultMessageStore store) { try { + Field serviceField = null; + if (store instanceof RocksDBMessageStore) { + serviceField = store.getClass().getSuperclass().getDeclaredField("reputMessageService"); + } else { + serviceField = store.getClass().getDeclaredField("reputMessageService"); + } - Field serviceField = store.getClass().getDeclaredField("reputMessageService"); serviceField.setAccessible(true); DefaultMessageStore.ReputMessageService reputService = - (DefaultMessageStore.ReputMessageService) serviceField.get(store); + (DefaultMessageStore.ReputMessageService) serviceField.get(store); Method method = DefaultMessageStore.ReputMessageService.class.getDeclaredMethod("isCommitLogAvailable"); method.setAccessible(true); diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java index 54174ac166c..fa8f41dbf84 100644 --- a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java @@ -36,7 +36,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; - +import org.rocksdb.RocksDBException; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.anyLong; @@ -114,7 +114,7 @@ public Boolean call() { } @Test - public void inSyncReplicasNums() throws IOException { + public void inSyncReplicasNums() throws IOException, RocksDBException { DefaultMessageStore messageStore = mockMessageStore(); doReturn(123L).when(messageStore).getMaxPhyOffset(); doReturn(123L).when(messageStore).getMasterFlushedOffset(); @@ -150,7 +150,7 @@ public Boolean call() throws Exception { } @Test - public void isSlaveOK() throws IOException { + public void isSlaveOK() throws IOException, RocksDBException { DefaultMessageStore messageStore = mockMessageStore(); doReturn(123L).when(messageStore).getMaxPhyOffset(); doReturn(123L).when(messageStore).getMasterFlushedOffset(); @@ -175,7 +175,8 @@ public Boolean call() throws Exception { } @Test - public void putRequest_SingleAck() throws IOException, ExecutionException, InterruptedException, TimeoutException { + public void putRequest_SingleAck() + throws IOException, ExecutionException, InterruptedException, TimeoutException, RocksDBException { CommitLog.GroupCommitRequest request = new CommitLog.GroupCommitRequest(124, 4000, 1); this.haService.putRequest(request); @@ -192,7 +193,8 @@ public void putRequest_SingleAck() throws IOException, ExecutionException, Inter } @Test - public void putRequest_MultipleAckAndRequests() throws IOException, ExecutionException, InterruptedException { + public void putRequest_MultipleAckAndRequests() + throws IOException, ExecutionException, InterruptedException, RocksDBException { CommitLog.GroupCommitRequest oneAck = new CommitLog.GroupCommitRequest(124, 4000, 2); this.haService.putRequest(oneAck); @@ -218,7 +220,7 @@ public void putRequest_MultipleAckAndRequests() throws IOException, ExecutionExc } @Test - public void getPush2SlaveMaxOffset() throws IOException { + public void getPush2SlaveMaxOffset() throws IOException, RocksDBException { DefaultMessageStore messageStore = mockMessageStore(); doReturn(123L).when(messageStore).getMaxPhyOffset(); doReturn(123L).when(messageStore).getMasterFlushedOffset(); @@ -256,7 +258,7 @@ private void setUpOneHAClient() throws IOException { this.haClientList.add(haClient); } - private DefaultMessageStore mockMessageStore() throws IOException { + private DefaultMessageStore mockMessageStore() throws IOException, RocksDBException { DefaultMessageStore messageStore = mock(DefaultMessageStore.class); BrokerConfig brokerConfig = mock(BrokerConfig.class); diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java index 27dcff14167..db5c5af4cd3 100644 --- a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java +++ b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java @@ -41,6 +41,7 @@ import org.junit.Assume; import org.junit.Ignore; import org.junit.Test; +import org.rocksdb.RocksDBException; import java.io.File; import java.net.InetAddress; @@ -180,7 +181,7 @@ public void init(int mappedFileSize, boolean allAckInSyncStateSet) throws Except private boolean changeMasterAndPutMessage(DefaultMessageStore master, MessageStoreConfig masterConfig, DefaultMessageStore slave, long slaveId, MessageStoreConfig slaveConfig, int epoch, String masterHaAddress, - int totalPutMessageNums) { + int totalPutMessageNums) throws RocksDBException { boolean flag = true; // Change role diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java index b7392cc4555..3c7b9b67fba 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java @@ -16,6 +16,9 @@ */ package org.apache.rocketmq.store.timer; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; import org.junit.Assert; import org.junit.Test; @@ -31,8 +34,11 @@ public void testTimingCount() { TimerMetrics first = new TimerMetrics(baseDir); Assert.assertTrue(first.load()); - first.addAndGet("AAA", 1000); - first.addAndGet("BBB", 2000); + MessageExt msg = new MessageExt(); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "AAA"); + first.addAndGet(msg, 1000); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "BBB"); + first.addAndGet(msg, 2000); Assert.assertEquals(1000, first.getTimingCount("AAA")); Assert.assertEquals(2000, first.getTimingCount("BBB")); long curr = System.currentTimeMillis(); diff --git a/test/pom.xml b/test/pom.xml index 8f25c35c971..168cbab0be2 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java index 496bd6da432..09c60c0b45f 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.test.client.rmq; +import java.util.List; import java.util.concurrent.CompletableFuture; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.AckCallback; @@ -140,6 +141,27 @@ public void onException(Throwable e) { return future; } + public CompletableFuture batchAckMessageAsync(String brokerAddr, String topic, String consumerGroup, + List extraInfoList) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.mqClientAPI.batchAckMessageAsync(brokerAddr, DEFAULT_TIMEOUT, new AckCallback() { + @Override + public void onSuccess(AckResult ackResult) { + future.complete(ackResult); + } + + @Override + public void onException(Throwable e) { + future.completeExceptionally(e); + } + }, topic, consumerGroup, extraInfoList); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + public CompletableFuture changeInvisibleTimeAsync(String brokerAddr, String brokerName, String topic, String consumerGroup, String extraInfo, long invisibleTime) { String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo); diff --git a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java index f3d105bc6b4..080b7e38528 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java @@ -28,7 +28,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; - import javax.annotation.Generated; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java index 952fbe3f5f5..2e29b95a5af 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java @@ -63,4 +63,10 @@ protected CompletableFuture popMessageAsync(long invisibleTime, int m brokerAddr, messageQueue, invisibleTime, maxNums, group, timeout, true, ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); } + + protected CompletableFuture popMessageAsync(long invisibleTime, int maxNums) { + return client.popMessageAsync( + brokerAddr, messageQueue, invisibleTime, maxNums, group, 3000, false, + ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); + } } diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java new file mode 100644 index 00000000000..ec9153ccc98 --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.client.consumer.pop; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.common.attribute.CQType; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.constant.ConsumeInitMode; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.test.base.IntegrationTestBase; +import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.test.client.rmq.RMQPopClient; +import org.apache.rocketmq.test.util.MQRandomUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; + +public class BatchAckIT extends BasePop { + + protected String topic; + protected String group; + protected RMQNormalProducer producer = null; + protected RMQPopClient client = null; + protected String brokerAddr; + protected MessageQueue messageQueue; + + @Before + public void setUp() { + brokerAddr = brokerController1.getBrokerAddr(); + topic = MQRandomUtils.getRandomTopic(); + group = initConsumerGroup(); + IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL); + producer = getProducer(NAMESRV_ADDR, topic); + client = getRMQPopClient(); + messageQueue = new MessageQueue(topic, BROKER1_NAME, -1); + } + + @After + public void tearDown() { + shutdown(); + } + + @Test + public void testBatchAckNormallyWithPopBuffer() throws Throwable { + brokerController1.getBrokerConfig().setEnablePopBufferMerge(true); + brokerController2.getBrokerConfig().setEnablePopBufferMerge(true); + + testBatchAck(() -> { + try { + return popMessageAsync().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + @Test + public void testBatchAckNormallyWithOutPopBuffer() throws Throwable { + brokerController1.getBrokerConfig().setEnablePopBufferMerge(false); + brokerController2.getBrokerConfig().setEnablePopBufferMerge(false); + + testBatchAck(() -> { + try { + return popMessageAsync().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + @Test + public void testBatchAckOrderly() throws Throwable { + testBatchAck(() -> { + try { + return popMessageOrderlyAsync().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + public void testBatchAck(Supplier popResultSupplier) throws Throwable { + // Send 10 messages but do not ack, let them enter the retry topic + producer.send(10); + AtomicInteger firstMsgRcvNum = new AtomicInteger(); + await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { + PopResult popResult = popResultSupplier.get(); + if (popResult.getPopStatus().equals(PopStatus.FOUND)) { + firstMsgRcvNum.addAndGet(popResult.getMsgFoundList().size()); + } + assertEquals(10, firstMsgRcvNum.get()); + }); + // sleep 6s, expect messages to enter the retry topic + TimeUnit.SECONDS.sleep(6); + + producer.send(20); + List extraInfoList = new ArrayList<>(); + await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { + PopResult popResult = popResultSupplier.get(); + if (popResult.getPopStatus().equals(PopStatus.FOUND)) { + for (MessageExt messageExt : popResult.getMsgFoundList()) { + extraInfoList.add(messageExt.getProperty(MessageConst.PROPERTY_POP_CK)); + } + } + assertEquals(30, extraInfoList.size()); + }); + + AckResult ackResult = client.batchAckMessageAsync(brokerAddr, topic, group, extraInfoList).get(); + assertEquals(AckStatus.OK, ackResult.getStatus()); + + // sleep 6s, expected that messages that have been acked will not be re-consumed + TimeUnit.SECONDS.sleep(6); + PopResult popResult = popResultSupplier.get(); + assertEquals(PopStatus.POLLING_NOT_FOUND, popResult.getPopStatus()); + } + + private CompletableFuture popMessageAsync() { + return client.popMessageAsync( + brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false, + ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); + } + + private CompletableFuture popMessageOrderlyAsync() { + return client.popMessageAsync( + brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false, + ConsumeInitMode.MIN, true, ExpressionType.TAG, "*", null); + } +} diff --git a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java index 7e3c7b871dc..9004b91db39 100644 --- a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.test.route; +import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; import org.apache.rocketmq.test.base.BaseConf; import org.apache.rocketmq.test.util.MQAdminTestUtils; @@ -111,4 +112,34 @@ public void testStaticTopicNotAffected() throws Exception { brokerController3.getBrokerConfig().setEnableSingleTopicRegister(false); namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(false); } + + @Test + public void testCreateOrUpdateTopic_EnableSplitRegistration() { + brokerController1.getBrokerConfig().setEnableSplitRegistration(true); + brokerController2.getBrokerConfig().setEnableSplitRegistration(true); + brokerController3.getBrokerConfig().setEnableSplitRegistration(true); + + String testTopic = "test-topic-"; + + for (int i = 0; i < 10; i++) { + TopicConfig topicConfig = new TopicConfig(testTopic + i, 8, 8); + brokerController1.getTopicConfigManager().updateTopicConfig(topicConfig); + brokerController2.getTopicConfigManager().updateTopicConfig(topicConfig); + brokerController3.getTopicConfigManager().updateTopicConfig(topicConfig); + } + + brokerController1.registerBrokerAll(false, true, true); + brokerController2.registerBrokerAll(false, true, true); + brokerController3.registerBrokerAll(false, true, true); + + for (int i = 0; i < 10; i++) { + TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic + i); + assertThat(route.getBrokerDatas()).hasSize(3); + assertThat(route.getQueueDatas()).hasSize(3); + } + + brokerController1.getBrokerConfig().setEnableSplitRegistration(false); + brokerController2.getBrokerConfig().setEnableSplitRegistration(false); + brokerController3.getBrokerConfig().setEnableSplitRegistration(false); + } } diff --git a/tieredstore/BUILD.bazel b/tieredstore/BUILD.bazel index 5b3885a4eae..8b6705ac283 100644 --- a/tieredstore/BUILD.bazel +++ b/tieredstore/BUILD.bazel @@ -36,9 +36,11 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_sdk", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:org_apache_commons_commons_lang3", "@maven//:org_apache_tomcat_annotations_api", "@maven//:com_alibaba_fastjson", + "@maven//:org_apache_rocketmq_rocketmq_rocksdb", ], ) diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml index c476040babd..b2ea40bf3a3 100644 --- a/tieredstore/pom.xml +++ b/tieredstore/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java index 1746190cdb1..766c559e9c8 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java @@ -48,6 +48,8 @@ import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager; import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant; import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; +import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicBlackListFilter; +import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicFilter; import org.apache.rocketmq.tieredstore.util.CQItemBufferUtil; import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; @@ -56,6 +58,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); + private TieredStoreTopicFilter topicFilter; private final String brokerName; private final MessageStore defaultStore; private final TieredMessageStoreConfig storeConfig; @@ -70,15 +73,15 @@ public TieredDispatcher(MessageStore defaultStore, TieredMessageStoreConfig stor this.defaultStore = defaultStore; this.storeConfig = storeConfig; this.brokerName = storeConfig.getBrokerName(); + this.topicFilter = new TieredStoreTopicBlackListFilter(); this.tieredFlatFileManager = TieredFlatFileManager.getInstance(storeConfig); this.dispatchRequestReadMap = new ConcurrentHashMap<>(); this.dispatchRequestWriteMap = new ConcurrentHashMap<>(); this.dispatchTaskLock = new ReentrantLock(); this.dispatchWriteLock = new ReentrantLock(); - this.initScheduleTask(); } - private void initScheduleTask() { + protected void initScheduleTask() { TieredStoreExecutor.commonScheduledExecutor.scheduleWithFixedDelay(() -> tieredFlatFileManager.deepCopyFlatFileToList().forEach(flatFile -> { if (!flatFile.getCompositeFlatFileLock().isLocked()) { @@ -87,6 +90,14 @@ private void initScheduleTask() { }), 30, 10, TimeUnit.SECONDS); } + public TieredStoreTopicFilter getTopicFilter() { + return topicFilter; + } + + public void setTopicFilter(TieredStoreTopicFilter topicFilter) { + this.topicFilter = topicFilter; + } + @Override public void dispatch(DispatchRequest request) { if (stopped) { @@ -94,7 +105,7 @@ public void dispatch(DispatchRequest request) { } String topic = request.getTopic(); - if (TieredStoreUtil.isSystemTopic(topic)) { + if (topicFilter != null && topicFilter.filterTopic(topic)) { return; } @@ -219,6 +230,10 @@ protected void dispatchFlatFile(CompositeQueueFlatFile flatFile) { return; } + if (topicFilter != null && topicFilter.filterTopic(flatFile.getMessageQueue().getTopic())) { + return; + } + if (flatFile.getDispatchOffset() == -1L) { return; } @@ -318,8 +333,7 @@ protected void dispatchFlatFile(CompositeQueueFlatFile flatFile) { continue; case FILE_CLOSED: tieredFlatFileManager.destroyCompositeFile(flatFile.getMessageQueue()); - logger.info("TieredDispatcher#dispatchFlatFile: file has been close and destroy, " + - "topic: {}, queueId: {}", topic, queueId); + logger.info("File has been closed and destroy, topic: {}, queueId: {}", topic, queueId); return; default: dispatchOffset--; diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java index 9a9a3e5a5cf..c948fa3fa17 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java @@ -273,15 +273,17 @@ public CompletableFuture getMessageFromCacheAsync(CompositeQue TieredStoreMetricsManager.cacheHit.add(resultWrapperList.size(), attributes); } - // if no cached message found and there is currently an inflight request, wait for the request to end before continuing + // If there are no messages in the cache and there are currently requests being pulled. + // We need to wait for the request to return before continuing. if (resultWrapperList.isEmpty() && waitInflightRequest) { - CompletableFuture future = flatFile.getInflightRequest(group, queueOffset, maxCount) - .getFuture(queueOffset); + CompletableFuture future = + flatFile.getInflightRequest(group, queueOffset, maxCount).getFuture(queueOffset); if (!future.isDone()) { Stopwatch stopwatch = Stopwatch.createStarted(); // to prevent starvation issues, only allow waiting for inflight request once return future.thenCompose(v -> { - LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: wait for inflight request cost: {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: wait for response cost: {}ms", + stopwatch.elapsed(TimeUnit.MILLISECONDS)); return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, false); }); } @@ -302,7 +304,8 @@ public CompletableFuture getMessageFromCacheAsync(CompositeQue // if cache hit, result will be returned immediately and asynchronously prefetch messages for later requests if (!resultWrapperList.isEmpty()) { - LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: cache hit: topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}", + LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: cache hit: " + + "topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}", mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, resultWrapperList.size()); prefetchMessage(flatFile, group, maxCount, lastGetOffset + 1); @@ -316,8 +319,10 @@ public CompletableFuture getMessageFromCacheAsync(CompositeQue } // if cache is miss, immediately pull messages - LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: topic: {}, queue: {}, queue offset: {}, max message num: {}", + LOGGER.info("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + + "topic: {}, queue: {}, queue offset: {}, max message num: {}", mq.getTopic(), mq.getQueueId(), queueOffset, maxCount); + CompletableFuture resultFuture; synchronized (flatFile) { int batchSize = maxCount * storeConfig.getReadAheadMinFactor(); @@ -453,42 +458,42 @@ public CompletableFuture getMessageFromTieredStoreAsync(Compos public CompletableFuture getMessageAsync( String group, String topic, int queueId, long queueOffset, int maxCount, final MessageFilter messageFilter) { + GetMessageResult result = new GetMessageResult(); CompositeQueueFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId)); + if (flatFile == null) { - GetMessageResult result = new GetMessageResult(); result.setNextBeginOffset(queueOffset); result.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE); return CompletableFuture.completedFuture(result); } - GetMessageResult result = new GetMessageResult(); - long minQueueOffset = flatFile.getConsumeQueueMinOffset(); - long maxQueueOffset = flatFile.getConsumeQueueCommitOffset(); - result.setMinOffset(minQueueOffset); - result.setMaxOffset(maxQueueOffset); + // Max queue offset means next message put position + result.setMinOffset(flatFile.getConsumeQueueMinOffset()); + result.setMaxOffset(flatFile.getConsumeQueueCommitOffset()); + + // Fill result according file offset. + // Offset range | Result | Fix to + // (-oo, 0] | no message | current offset + // (0, min) | too small | min offset + // [min, max) | correct | + // [max, max] | overflow one | max offset + // (max, +oo) | overflow badly | max offset - if (flatFile.getConsumeQueueCommitOffset() <= 0) { + if (result.getMaxOffset() <= 0) { result.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE); result.setNextBeginOffset(queueOffset); return CompletableFuture.completedFuture(result); - } - - // request range | result - // (0, min) | too small - // [min, max) | correct - // [max, max] | overflow one - // (max, +oo) | overflow badly - if (queueOffset < minQueueOffset) { + } else if (queueOffset < result.getMinOffset()) { result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL); - result.setNextBeginOffset(flatFile.getConsumeQueueMinOffset()); + result.setNextBeginOffset(result.getMinOffset()); return CompletableFuture.completedFuture(result); - } else if (queueOffset == maxQueueOffset) { + } else if (queueOffset == result.getMaxOffset()) { result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE); - result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset()); + result.setNextBeginOffset(result.getMaxOffset()); return CompletableFuture.completedFuture(result); - } else if (queueOffset > maxQueueOffset) { + } else if (queueOffset > result.getMaxOffset()) { result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY); - result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset()); + result.setNextBeginOffset(result.getMaxOffset()); return CompletableFuture.completedFuture(result); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java index 5240ac8e9ea..edaa5d19f61 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java @@ -16,17 +16,14 @@ */ package org.apache.rocketmq.tieredstore; -import com.google.common.base.Stopwatch; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.ViewBuilder; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; + +import com.google.common.base.Stopwatch; + import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.MixAll; @@ -55,6 +52,12 @@ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.ViewBuilder; + public class TieredMessageStore extends AbstractPluginMessageStore { protected static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); @@ -90,6 +93,7 @@ public boolean load() { boolean loadNextStore = next.load(); boolean result = loadFlatFile && loadNextStore; if (result) { + dispatcher.initScheduleTask(); dispatcher.start(); } return result; @@ -99,11 +103,11 @@ public TieredMessageStoreConfig getStoreConfig() { return storeConfig; } - public boolean viaTieredStorage(String topic, int queueId, long offset) { - return viaTieredStorage(topic, queueId, offset, 1); + public boolean fetchFromCurrentStore(String topic, int queueId, long offset) { + return fetchFromCurrentStore(topic, queueId, offset, 1); } - public boolean viaTieredStorage(String topic, int queueId, long offset, int batchSize) { + public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int batchSize) { TieredMessageStoreConfig.TieredStorageLevel deepStorageLevel = storeConfig.getTieredStorageLevel(); if (deepStorageLevel.check(TieredMessageStoreConfig.TieredStorageLevel.FORCE)) { @@ -146,8 +150,15 @@ public GetMessageResult getMessage(String group, String topic, int queueId, long public CompletableFuture getMessageAsync(String group, String topic, int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { - if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) { - logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); + // For system topic, force reading from local store + if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { + return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); + } + + if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { + logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); + } else { + logger.trace("GetMessageAsync from next store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); } @@ -155,6 +166,7 @@ public CompletableFuture getMessageAsync(String group, String return fetcher .getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter) .thenApply(result -> { + Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder() .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_MESSAGE) .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic) @@ -163,26 +175,19 @@ public CompletableFuture getMessageAsync(String group, String TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); if (result.getStatus() == GetMessageStatus.OFFSET_FOUND_NULL || - result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_ONE || - result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_BADLY) { + result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) { TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes); - logger.debug("GetMessageAsync not found then try back to next store, result: {}, " + + logger.debug("GetMessageAsync not found, then back to next store, result: {}, " + "topic: {}, queue: {}, queue offset: {}, offset range: {}-{}", result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset()); return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); } } - // system topic - if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { - if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { - return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); - } - } - if (result.getStatus() != GetMessageStatus.FOUND && + result.getStatus() != GetMessageStatus.NO_MATCHED_LOGIC_QUEUE && result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_ONE && result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_BADLY) { logger.warn("GetMessageAsync not found and message is not in next store, result: {}, " + @@ -198,18 +203,22 @@ public CompletableFuture getMessageAsync(String group, String TieredStoreMetricsManager.messagesOutTotal.add(result.getMessageCount(), messagesOutAttributes); } - // fix min or max offset according next store + // Fix min or max offset according next store at last long minOffsetInQueue = next.getMinOffsetInQueue(topic, queueId); if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) { result.setMinOffset(minOffsetInQueue); } - long maxOffsetInQueue = next.getMaxOffsetInQueue(topic, queueId); - if (maxOffsetInQueue >= 0 && maxOffsetInQueue > result.getMaxOffset()) { - result.setMaxOffset(maxOffsetInQueue); - } + + // In general, the local cq offset is slightly greater than the commit offset in read message, + // so there is no need to update the maximum offset to the local cq offset here, + // otherwise it will cause repeated consumption after next begin offset over commit offset. + + logger.trace("GetMessageAsync result, group: {}, topic: {}, queueId: {}, offset: {}, count:{}, {}", + group, topic, queueId, offset, maxMsgNums, result); + return result; }).exceptionally(e -> { - logger.error("GetMessageAsync from tiered store failed: ", e); + logger.error("GetMessageAsync from tiered store failed", e); return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); }); } @@ -251,7 +260,7 @@ public CompletableFuture getEarliestMessageTimeAsync(String topic, int que .build(); TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); if (time < 0) { - logger.debug("TieredMessageStore#getEarliestMessageTimeAsync: get earliest message time failed, try to get earliest message time from next store: topic: {}, queue: {}", + logger.debug("GetEarliestMessageTimeAsync failed, try to get earliest message time from next store: topic: {}, queue: {}", topic, queueId); return finalNextEarliestMessageTime != Long.MAX_VALUE ? finalNextEarliestMessageTime : -1; } @@ -262,7 +271,7 @@ public CompletableFuture getEarliestMessageTimeAsync(String topic, int que @Override public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, long consumeQueueOffset) { - if (viaTieredStorage(topic, queueId, consumeQueueOffset)) { + if (fetchFromCurrentStore(topic, queueId, consumeQueueOffset)) { Stopwatch stopwatch = Stopwatch.createStarted(); return fetcher.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset) .thenApply(time -> { @@ -272,7 +281,7 @@ public CompletableFuture getMessageStoreTimeStampAsync(String topic, int q .build(); TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); if (time == -1) { - logger.debug("TieredMessageStore#getMessageStoreTimeStampAsync: get message time failed, try to get message time from next store: topic: {}, queue: {}, queue offset: {}", + logger.debug("GetEarliestMessageTimeAsync failed, try to get message time from next store, topic: {}, queue: {}, queue offset: {}", topic, queueId, consumeQueueOffset); return next.getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java index 6dd0e8846ea..65d586f43dd 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java @@ -20,10 +20,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.utils.ThreadUtils; public class TieredStoreExecutor { @@ -43,20 +43,20 @@ public class TieredStoreExecutor { public static ExecutorService compactIndexFileExecutor; public static void init() { - commonScheduledExecutor = new ScheduledThreadPoolExecutor( + commonScheduledExecutor = ThreadUtils.newScheduledThreadPool( Math.max(4, Runtime.getRuntime().availableProcessors()), new ThreadFactoryImpl("TieredCommonExecutor_")); - commitExecutor = new ScheduledThreadPoolExecutor( + commitExecutor = ThreadUtils.newScheduledThreadPool( Math.max(16, Runtime.getRuntime().availableProcessors() * 4), new ThreadFactoryImpl("TieredCommitExecutor_")); - cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor( + cleanExpiredFileExecutor = ThreadUtils.newScheduledThreadPool( Math.max(4, Runtime.getRuntime().availableProcessors()), new ThreadFactoryImpl("TieredCleanFileExecutor_")); dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); - dispatchExecutor = new ThreadPoolExecutor( + dispatchExecutor = ThreadUtils.newThreadPoolExecutor( Math.max(2, Runtime.getRuntime().availableProcessors()), Math.max(16, Runtime.getRuntime().availableProcessors() * 4), 1000 * 60, @@ -66,7 +66,7 @@ public static void init() { new ThreadPoolExecutor.DiscardOldestPolicy()); fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); - fetchDataExecutor = new ThreadPoolExecutor( + fetchDataExecutor = ThreadUtils.newThreadPoolExecutor( Math.max(16, Runtime.getRuntime().availableProcessors() * 4), Math.max(64, Runtime.getRuntime().availableProcessors() * 8), 1000 * 60, @@ -75,7 +75,7 @@ public static void init() { new ThreadFactoryImpl("TieredFetchExecutor_")); compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); - compactIndexFileExecutor = new ThreadPoolExecutor( + compactIndexFileExecutor = ThreadUtils.newThreadPoolExecutor( 1, 1, 1000 * 60, diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java index 426c4e09d34..d96eb6e8f3c 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.tieredstore.file; -import com.alibaba.fastjson.JSON; import com.google.common.annotations.VisibleForTesting; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -25,13 +24,13 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.tieredstore.common.AppendResult; @@ -43,7 +42,6 @@ import org.apache.rocketmq.tieredstore.provider.FileSegmentAllocator; import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; -import org.apache.rocketmq.common.BoundaryType; public class TieredFlatFile { @@ -177,7 +175,10 @@ protected void recoverMetadata() { } } - private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) { + /** + * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full + */ + public void updateFileSegment(TieredFileSegment fileSegment) { FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); @@ -186,45 +187,24 @@ private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fil if (metadata == null) { metadata = new FileSegmentMetadata( this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); - metadata.setCreateTimestamp(fileSegment.getMinTimestamp()); - metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); - metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); - if (fileSegment.isClosed()) { - metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); - } - this.tieredMetadataStore.updateFileSegment(metadata); + metadata.setCreateTimestamp(System.currentTimeMillis()); } - return metadata; - } - - /** - * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full - */ - public void updateFileSegment(TieredFileSegment fileSegment) { - FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment); - if (segmentMetadata.getStatus() == FileSegmentMetadata.STATUS_NEW - && fileSegment.isFull() - && !fileSegment.needCommit()) { + metadata.setSize(fileSegment.getCommitPosition()); + metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); + metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); - segmentMetadata.markSealed(); + if (fileSegment.isFull() && !fileSegment.needCommit()) { + if (metadata.getStatus() == FileSegmentMetadata.STATUS_NEW) { + metadata.markSealed(); + } } if (fileSegment.isClosed()) { - segmentMetadata.setStatus(FileSegmentMetadata.STATUS_DELETED); + metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); } - segmentMetadata.setSize(fileSegment.getCommitPosition()); - segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp()); - - FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( - this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); - - if (!Objects.equals(metadata, segmentMetadata)) { - this.tieredMetadataStore.updateFileSegment(segmentMetadata); - logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}", - segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata)); - } + this.tieredMetadataStore.updateFileSegment(metadata); } private void checkAndFixFileSize() { @@ -365,7 +345,10 @@ protected TieredFileSegment getFileByTime(long timestamp, BoundaryType boundaryT if (!segmentList.isEmpty()) { return boundaryType == BoundaryType.UPPER ? segmentList.get(0) : segmentList.get(segmentList.size() - 1); } - return fileSegmentList.isEmpty() ? null : fileSegmentList.get(fileSegmentList.size() - 1); + if (fileSegmentList.isEmpty()) { + return null; + } + return boundaryType == BoundaryType.UPPER ? fileSegmentList.get(fileSegmentList.size() - 1) : fileSegmentList.get(0); } finally { fileSegmentLock.readLock().unlock(); } @@ -595,6 +578,9 @@ public void destroy() { logger.error("TieredFlatFile#destroy: mark file segment: {} is deleted failed", fileSegment.getPath(), e); } fileSegment.destroyFile(); + if (!fileSegment.exists()) { + tieredMetadataStore.deleteFileSegment(filePath, fileType, fileSegment.getBaseOffset()); + } } fileSegmentList.clear(); needCommitFileSegmentList.clear(); diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java index e9ae4a5a521..087ea8c9ce6 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java @@ -134,21 +134,19 @@ public void doCommit() { public void doCleanExpiredFile() { long expiredTimeStamp = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); - Random random = new Random(); for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) { - int delay = random.nextInt(storeConfig.getMaxCommitJitter()); - TieredStoreExecutor.cleanExpiredFileExecutor.schedule(() -> { - flatFile.getCompositeFlatFileLock().lock(); + TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> { try { + flatFile.getCompositeFlatFileLock().lock(); flatFile.cleanExpiredFile(expiredTimeStamp); flatFile.destroyExpiredFile(); - if (flatFile.getConsumeQueueBaseOffset() == -1) { - destroyCompositeFile(flatFile.getMessageQueue()); - } + } catch (Throwable t) { + logger.error("Do Clean expired file error, topic={}, queueId={}", + flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t); } finally { flatFile.getCompositeFlatFileLock().unlock(); } - }, delay, TimeUnit.MILLISECONDS); + }); } if (indexFile != null) { indexFile.cleanExpiredFile(expiredTimeStamp); @@ -218,8 +216,13 @@ public void recoverTieredFlatFile() { storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId())); queueCount.incrementAndGet(); }); - logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", - topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); + + if (queueCount.get() == 0L) { + metadataStore.deleteTopic(topicMetadata.getTopic()); + } else { + logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", + topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); + } } catch (Exception e) { logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e); } finally { diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java index 50beb01ae4b..eda5e010657 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.tieredstore.file; +import com.google.common.annotations.VisibleForTesting; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -99,7 +100,7 @@ protected TieredIndexFile(TieredFileAllocator fileQueueFactory, String filePath) this::doScheduleTask, 10, 10, TimeUnit.SECONDS); } - private void doScheduleTask() { + protected void doScheduleTask() { try { curFileLock.lock(); try { @@ -145,6 +146,11 @@ private void initIndexFileHeader(MappedFile mappedFile) { } } + @VisibleForTesting + public MappedFile getPreMappedFile() { + return preMappedFile; + } + private void initFile() throws IOException { curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); initIndexFileHeader(curMappedFile); @@ -156,19 +162,26 @@ private void initFile() throws IOException { if (isFileSealed(curMappedFile)) { if (preFileExists) { - preFile.delete(); + if (preFile.delete()) { + logger.info("Pre IndexFile deleted success", preFilepath); + } else { + logger.error("Pre IndexFile deleted failed", preFilepath); + } } boolean rename = curMappedFile.renameTo(preFilepath); if (rename) { preMappedFile = curMappedFile; curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); + initIndexFileHeader(curMappedFile); preFileExists = true; } } + if (preFileExists) { synchronized (TieredIndexFile.class) { if (inflightCompactFuture.isDone()) { - inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null); + inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit( + new CompactTask(storeConfig, preMappedFile, flatFile), null); } } } @@ -261,7 +274,8 @@ private AppendResult putKey(MessageQueue mq, int topicId, int hashCode, long off } } - public CompletableFuture>> queryAsync(String topic, String key, long beginTime, long endTime) { + public CompletableFuture>> queryAsync(String topic, String key, long beginTime, + long endTime) { int hashCode = indexKeyHashMethod(buildKey(topic, key)); int slotPosition = hashCode % maxHashSlotNum; List fileSegmentList = flatFile.getFileListByTime(beginTime, endTime); @@ -355,7 +369,7 @@ static class CompactTask implements Runnable { private final int fileMaxSize; private MappedFile originFile; private TieredFlatFile fileQueue; - private final MappedFile compactFile; + private MappedFile compactFile; public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile, TieredFlatFile fileQueue) throws IOException { @@ -381,6 +395,17 @@ public void run() { } catch (Throwable throwable) { logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable); } + + try { + if (originFile != null) { + originFile.destroy(-1); + } + if (compactFile != null) { + compactFile.destroy(-1); + } + } catch (Throwable throwable) { + logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable); + } } public void compact() { @@ -396,6 +421,8 @@ public void compact() { fileQueue.commit(true); compactFile.destroy(-1); originFile.destroy(-1); + compactFile = null; + originFile = null; } private void buildCompactFile() { @@ -414,6 +441,7 @@ private void buildCompactFile() { if (slotValue != -1) { int indexTotalSize = 0; int indexPosition = slotValue; + while (indexPosition >= 0 && indexPosition < maxIndexNum) { int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE; diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java index 5062c7d9e7c..32911a6e898 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java @@ -16,14 +16,11 @@ */ package org.apache.rocketmq.tieredstore.provider; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Stopwatch; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -35,8 +32,8 @@ import org.apache.rocketmq.tieredstore.file.TieredCommitLog; import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue; import org.apache.rocketmq.tieredstore.file.TieredIndexFile; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory; import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; @@ -50,22 +47,23 @@ public abstract class TieredFileSegment implements Comparable protected final TieredMessageStoreConfig storeConfig; private final long maxSize; - private final ReentrantLock bufferLock; - private final Semaphore commitLock; + private final ReentrantLock bufferLock = new ReentrantLock(); + private final Semaphore commitLock = new Semaphore(1); - private volatile boolean full; - private volatile boolean closed; + private volatile boolean full = false; + private volatile boolean closed = false; - private volatile long minTimestamp; - private volatile long maxTimestamp; - private volatile long commitPosition; - private volatile long appendPosition; + private volatile long minTimestamp = Long.MAX_VALUE; + private volatile long maxTimestamp = Long.MAX_VALUE; + private volatile long commitPosition = 0L; + private volatile long appendPosition = 0L; // only used in commitLog - private volatile long dispatchCommitOffset = 0; + private volatile long dispatchCommitOffset = 0L; private ByteBuffer codaBuffer; - private List uploadBufferList = new ArrayList<>(); + private List bufferList = new ArrayList<>(); + private FileSegmentInputStream fileSegmentInputStream; private CompletableFuture flightCommitRequest = CompletableFuture.completedFuture(false); public TieredFileSegment(TieredMessageStoreConfig storeConfig, @@ -75,21 +73,13 @@ public TieredFileSegment(TieredMessageStoreConfig storeConfig, this.fileType = fileType; this.filePath = filePath; this.baseOffset = baseOffset; - - this.closed = false; - this.bufferLock = new ReentrantLock(); - this.commitLock = new Semaphore(1); - - this.commitPosition = 0L; - this.appendPosition = 0L; - this.minTimestamp = Long.MAX_VALUE; - this.maxTimestamp = Long.MAX_VALUE; - - // The max segment size of a file is determined by the file type - this.maxSize = getMaxSizeAccordingFileType(storeConfig); + this.maxSize = getMaxSizeByFileType(); } - private long getMaxSizeAccordingFileType(TieredMessageStoreConfig storeConfig) { + /** + * The max segment size of a file is determined by the file type + */ + protected long getMaxSizeByFileType() { switch (fileType) { case COMMIT_LOG: return storeConfig.getTieredStoreCommitLogMaxSize(); @@ -184,39 +174,23 @@ public void initPosition(long pos) { this.appendPosition = pos; } - private List rollingUploadBuffer() { + private List borrowBuffer() { bufferLock.lock(); try { - List tmp = uploadBufferList; - uploadBufferList = new ArrayList<>(); + List tmp = bufferList; + bufferList = new ArrayList<>(); return tmp; } finally { bufferLock.unlock(); } } - private void sendBackBuffer(TieredFileSegmentInputStream inputStream) { - bufferLock.lock(); - try { - List tmpBufferList = inputStream.getUploadBufferList(); - for (ByteBuffer buffer : tmpBufferList) { - buffer.rewind(); - } - tmpBufferList.addAll(uploadBufferList); - uploadBufferList = tmpBufferList; - if (inputStream.getCodaBuffer() != null) { - codaBuffer.rewind(); - } - } finally { - bufferLock.unlock(); - } - } - @SuppressWarnings("NonAtomicOperationOnVolatileField") - public AppendResult append(ByteBuffer byteBuf, long timeStamp) { + public AppendResult append(ByteBuffer byteBuf, long timestamp) { if (closed) { return AppendResult.FILE_CLOSED; } + bufferLock.lock(); try { if (full || codaBuffer != null) { @@ -227,7 +201,8 @@ public AppendResult append(ByteBuffer byteBuf, long timeStamp) { minTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_BEGIN_TIME_STAMP_POSITION); maxTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_END_TIME_STAMP_POSITION); appendPosition += byteBuf.remaining(); - uploadBufferList.add(byteBuf); + // IndexFile is large and not change after compaction, no need deep copy + bufferList.add(byteBuf); setFull(); return AppendResult.SUCCESS; } @@ -236,23 +211,34 @@ public AppendResult append(ByteBuffer byteBuf, long timeStamp) { setFull(); return AppendResult.FILE_FULL; } - if (uploadBufferList.size() > storeConfig.getTieredStoreGroupCommitCount() + + if (bufferList.size() > storeConfig.getTieredStoreGroupCommitCount() || appendPosition - commitPosition > storeConfig.getTieredStoreGroupCommitSize()) { commitAsync(); } - if (uploadBufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) { - logger.debug("TieredFileSegment#append: buffer full: file: {}, upload buffer size: {}", - getPath(), uploadBufferList.size()); + + if (bufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) { + logger.debug("File segment append buffer full, file: {}, buffer size: {}, pending bytes: {}", + getPath(), bufferList.size(), appendPosition - commitPosition); return AppendResult.BUFFER_FULL; } - if (timeStamp != Long.MAX_VALUE) { - maxTimestamp = timeStamp; + + if (timestamp != Long.MAX_VALUE) { + maxTimestamp = timestamp; if (minTimestamp == Long.MAX_VALUE) { - minTimestamp = timeStamp; + minTimestamp = timestamp; } } + appendPosition += byteBuf.remaining(); - uploadBufferList.add(byteBuf); + + // deep copy buffer + ByteBuffer byteBuffer = ByteBuffer.allocateDirect(byteBuf.remaining()); + byteBuffer.put(byteBuf); + byteBuffer.flip(); + byteBuf.rewind(); + + bufferList.add(byteBuffer); return AppendResult.SUCCESS; } finally { bufferLock.unlock(); @@ -267,7 +253,6 @@ public long getAppendPosition() { return appendPosition; } - @VisibleForTesting public void setAppendPosition(long appendPosition) { this.appendPosition = appendPosition; } @@ -333,6 +318,8 @@ public boolean commit() { if (closed) { return false; } + // result is false when we send real commit request + // use join for wait flight request done Boolean result = commitAsync().join(); if (!result) { result = flightCommitRequest.join(); @@ -340,92 +327,156 @@ public boolean commit() { return result; } + private void releaseCommitLock() { + if (commitLock.availablePermits() == 0) { + commitLock.release(); + } else { + logger.error("[Bug] FileSegmentCommitAsync, lock is already released: available permits: {}", + commitLock.availablePermits()); + } + } + + private void updateDispatchCommitOffset(List bufferList) { + if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) { + dispatchCommitOffset = + MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1)); + } + } + + /** + * @return false: commit, true: no commit operation + */ @SuppressWarnings("NonAtomicOperationOnVolatileField") public CompletableFuture commitAsync() { if (closed) { return CompletableFuture.completedFuture(false); } - Stopwatch stopwatch = Stopwatch.createStarted(); + if (!needCommit()) { return CompletableFuture.completedFuture(true); } - try { - int permits = commitLock.drainPermits(); - if (permits <= 0) { - return CompletableFuture.completedFuture(false); - } - } catch (Exception e) { + + if (commitLock.drainPermits() <= 0) { return CompletableFuture.completedFuture(false); } - List bufferList = rollingUploadBuffer(); - int bufferSize = 0; - for (ByteBuffer buffer : bufferList) { - bufferSize += buffer.remaining(); - } - if (codaBuffer != null) { - bufferSize += codaBuffer.remaining(); - } - if (bufferSize == 0) { - return CompletableFuture.completedFuture(true); - } - TieredFileSegmentInputStream inputStream = TieredFileSegmentInputStreamFactory.build( - fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize); - int finalBufferSize = bufferSize; + try { - flightCommitRequest = commit0(inputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX) + if (fileSegmentInputStream != null) { + long fileSize = this.getSize(); + if (fileSize == -1L) { + logger.error("Get commit position error before commit, Commit: %d, Expect: %d, Current Max: %d, FileName: %s", + commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath()); + releaseCommitLock(); + return CompletableFuture.completedFuture(false); + } else { + if (correctPosition(fileSize, null)) { + updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); + fileSegmentInputStream = null; + } + } + } + + int bufferSize; + if (fileSegmentInputStream != null) { + bufferSize = fileSegmentInputStream.available(); + } else { + List bufferList = borrowBuffer(); + bufferSize = bufferList.stream().mapToInt(ByteBuffer::remaining).sum() + + (codaBuffer != null ? codaBuffer.remaining() : 0); + if (bufferSize == 0) { + releaseCommitLock(); + return CompletableFuture.completedFuture(true); + } + fileSegmentInputStream = FileSegmentInputStreamFactory.build( + fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize); + } + + return flightCommitRequest = this + .commit0(fileSegmentInputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX) .thenApply(result -> { if (result) { - if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) { - dispatchCommitOffset = MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1)); - } - commitPosition += finalBufferSize; + updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); + commitPosition += bufferSize; + fileSegmentInputStream = null; return true; - } - sendBackBuffer(inputStream); - return false; - }) - .exceptionally(e -> handleCommitException(inputStream, e)) - .whenComplete((result, e) -> { - if (commitLock.availablePermits() == 0) { - logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize); - commitLock.release(); } else { - logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits()); + fileSegmentInputStream.rewind(); + return false; } - }); - return flightCommitRequest; + }) + .exceptionally(this::handleCommitException) + .whenComplete((result, e) -> releaseCommitLock()); + } catch (Exception e) { - handleCommitException(inputStream, e); - if (commitLock.availablePermits() == 0) { - logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize); - commitLock.release(); - } else { - logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits()); - } + handleCommitException(e); + releaseCommitLock(); } return CompletableFuture.completedFuture(false); } - private boolean handleCommitException(TieredFileSegmentInputStream inputStream, Throwable e) { + private long getCorrectFileSize(Throwable throwable) { + if (throwable instanceof TieredStoreException) { + long fileSize = ((TieredStoreException) throwable).getPosition(); + if (fileSize > 0) { + return fileSize; + } + } + return getSize(); + } + + private boolean handleCommitException(Throwable e) { + // Get root cause here Throwable cause = e.getCause() != null ? e.getCause() : e; - sendBackBuffer(inputStream); - long realSize = 0; - if (cause instanceof TieredStoreException && ((TieredStoreException) cause).getPosition() > 0) { - realSize = ((TieredStoreException) cause).getPosition(); + long fileSize = this.getCorrectFileSize(cause); + + if (fileSize == -1L) { + logger.error("Get commit position error, Commit: %d, Expect: %d, Current Max: %d, FileName: %s", + commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath()); + fileSegmentInputStream.rewind(); + return false; + } + + if (correctPosition(fileSize, cause)) { + updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); + fileSegmentInputStream = null; + return true; + } else { + fileSegmentInputStream.rewind(); + return false; } - if (realSize <= 0) { - realSize = getSize(); + } + + /** + * return true to clear buffer + */ + private boolean correctPosition(long fileSize, Throwable throwable) { + + // Current we have three offsets here: commit offset, expect offset, file size. + // We guarantee that the commit offset is less than or equal to the expect offset. + // Max offset will increase because we can continuously put in new buffers + String handleInfo = throwable == null ? "before commit" : "after commit"; + long expectPosition = commitPosition + fileSegmentInputStream.getContentLength(); + + String offsetInfo = String.format("Correct Commit Position, %s, result=[{}], " + + "Commit: %d, Expect: %d, Current Max: %d, FileSize: %d, FileName: %s", + handleInfo, commitPosition, expectPosition, appendPosition, fileSize, this.getPath()); + + // We are believing that the file size returned by the server is correct, + // can reset the commit offset to the file size reported by the storage system. + if (fileSize == expectPosition) { + logger.info(offsetInfo, "Success", throwable); + commitPosition = fileSize; + return true; } - if (realSize > 0 && realSize > commitPosition) { - logger.error("TieredFileSegment#handleCommitException: commit failed: file: {}, try to fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause); - // TODO check if this diff part is uploaded to backend storage - long diff = appendPosition - commitPosition; - commitPosition = realSize; - appendPosition = realSize + diff; - // TODO check if appendPosition is large than maxOffset - } else if (realSize < commitPosition) { - logger.error("[Bug]TieredFileSegment#handleCommitException: commit failed: file: {}, can not fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause); + + if (fileSize < commitPosition) { + logger.error(offsetInfo, "FileSizeIncorrect", throwable); + } else if (fileSize == commitPosition) { + logger.warn(offsetInfo, "CommitFailed", throwable); + } else if (fileSize > commitPosition) { + logger.warn(offsetInfo, "PartialSuccess", throwable); } + commitPosition = fileSize; return false; } } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java index 5a0ca25f592..0db3eaf8f44 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java @@ -18,7 +18,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; public interface TieredStoreProvider { @@ -30,7 +30,9 @@ public interface TieredStoreProvider { String getPath(); /** - * Get file size in backend file system + * Get the real length of the file. + * Return 0 if the file does not exist, + * Return -1 if system get size failed. * * @return file real size */ @@ -71,5 +73,5 @@ public interface TieredStoreProvider { * @param append try to append or create a new file * @return put result, true if data successfully write; false otherwise */ - CompletableFuture commit0(TieredFileSegmentInputStream inputStream,long position, int length, boolean append); + CompletableFuture commit0(FileSegmentInputStream inputStream,long position, int length, boolean append); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java new file mode 100644 index 00000000000..f8bf165bc11 --- /dev/null +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tieredstore.provider; + +import java.util.HashSet; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + +public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter { + + private final Set topicBlackSet; + + public TieredStoreTopicBlackListFilter() { + this.topicBlackSet = new HashSet<>(); + } + + @Override + public boolean filterTopic(String topicName) { + if (StringUtils.isBlank(topicName)) { + return true; + } + return TieredStoreUtil.isSystemTopic(topicName) || topicBlackSet.contains(topicName); + } + + @Override + public void addTopicToBlackList(String topicName) { + this.topicBlackSet.add(topicName); + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java similarity index 55% rename from client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java index 80188832eb8..f983ed6e961 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java @@ -15,23 +15,11 @@ * limitations under the License. */ -package org.apache.rocketmq.client.impl; +package org.apache.rocketmq.tieredstore.provider; -import org.apache.rocketmq.remoting.InvokeCallback; -import org.apache.rocketmq.remoting.netty.ResponseFuture; +public interface TieredStoreTopicFilter { -public abstract class BaseInvokeCallback implements InvokeCallback { - private final MQClientAPIImpl mqClientAPI; + boolean filterTopic(String topicName); - public BaseInvokeCallback(MQClientAPIImpl mqClientAPI) { - this.mqClientAPI = mqClientAPI; - } - - @Override - public void operationComplete(final ResponseFuture responseFuture) { - mqClientAPI.execRpcHooksAfterRequest(responseFuture); - onComplete(responseFuture); - } - - public abstract void onComplete(final ResponseFuture responseFuture); + void addTopicToBlackList(String topicName); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java index 52be90b1dfa..7e949cb28cc 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java @@ -36,7 +36,7 @@ import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor; import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_FILE_TYPE; @@ -184,7 +184,7 @@ public CompletableFuture read0(long position, int length) { @Override public CompletableFuture commit0( - TieredFileSegmentInputStream inputStream, long position, int length, boolean append) { + FileSegmentInputStream inputStream, long position, int length, boolean append) { Stopwatch stopwatch = Stopwatch.createStarted(); AttributesBuilder attributesBuilder = newAttributesBuilder() diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java similarity index 88% rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java index c70bb76562a..13b6e0ef9c9 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.rocketmq.tieredstore.provider.inputstream; +package org.apache.rocketmq.tieredstore.provider.stream; import java.io.IOException; import java.nio.ByteBuffer; @@ -23,20 +23,23 @@ import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; -public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { +public class CommitLogInputStream extends FileSegmentInputStream { /** * commitLogOffset is the real physical offset of the commitLog buffer which is being read */ + private final long startCommitLogOffset; + private long commitLogOffset; private final ByteBuffer codaBuffer; private long markCommitLogOffset = -1; - public TieredCommitLogInputStream(FileSegmentType fileType, long startOffset, + public CommitLogInputStream(FileSegmentType fileType, long startOffset, List uploadBufferList, ByteBuffer codaBuffer, int contentLength) { super(fileType, uploadBufferList, contentLength); + this.startCommitLogOffset = startOffset; this.commitLogOffset = startOffset; this.codaBuffer = codaBuffer; } @@ -53,6 +56,15 @@ public synchronized void reset() throws IOException { this.commitLogOffset = markCommitLogOffset; } + @Override + public synchronized void rewind() { + super.rewind(); + this.commitLogOffset = this.startCommitLogOffset; + if (this.codaBuffer != null) { + this.codaBuffer.rewind(); + } + } + @Override public ByteBuffer getCodaBuffer() { return this.codaBuffer; @@ -64,17 +76,17 @@ public int read() { return -1; } readPosition++; - if (curReadBufferIndex >= uploadBufferList.size()) { + if (curReadBufferIndex >= bufferList.size()) { return readCoda(); } int res; if (readPosInCurBuffer >= curBuffer.remaining()) { curReadBufferIndex++; - if (curReadBufferIndex >= uploadBufferList.size()) { + if (curReadBufferIndex >= bufferList.size()) { readPosInCurBuffer = 0; return readCoda(); } - curBuffer = uploadBufferList.get(curReadBufferIndex); + curBuffer = bufferList.get(curReadBufferIndex); commitLogOffset += readPosInCurBuffer; readPosInCurBuffer = 0; } @@ -119,9 +131,9 @@ public int read(byte[] b, int off, int len) { int posInCurBuffer = readPosInCurBuffer; long curCommitLogOffset = commitLogOffset; ByteBuffer curBuf = curBuffer; - while (needRead > 0 && bufIndex <= uploadBufferList.size()) { + while (needRead > 0 && bufIndex <= bufferList.size()) { int readLen, remaining, realReadLen = 0; - if (bufIndex == uploadBufferList.size()) { + if (bufIndex == bufferList.size()) { // read from coda buffer remaining = codaBuffer.remaining() - posInCurBuffer; readLen = Math.min(remaining, needRead); @@ -137,7 +149,7 @@ public int read(byte[] b, int off, int len) { } remaining = curBuf.remaining() - posInCurBuffer; readLen = Math.min(remaining, needRead); - curBuf = uploadBufferList.get(bufIndex); + curBuf = bufferList.get(bufIndex); if (posInCurBuffer < MessageBufferUtil.PHYSICAL_OFFSET_POSITION) { realReadLen = Math.min(MessageBufferUtil.PHYSICAL_OFFSET_POSITION - posInCurBuffer, readLen); // read from commitLog buffer diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java similarity index 77% rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java index e1758ca934d..9e9d5135cd7 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java @@ -15,15 +15,16 @@ * limitations under the License. */ -package org.apache.rocketmq.tieredstore.provider.inputstream; +package org.apache.rocketmq.tieredstore.provider.stream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.apache.rocketmq.tieredstore.common.FileSegmentType; -public class TieredFileSegmentInputStream extends InputStream { +public class FileSegmentInputStream extends InputStream { /** * file type, can be commitlog, consume queue or indexfile now @@ -33,7 +34,7 @@ public class TieredFileSegmentInputStream extends InputStream { /** * hold bytebuffer */ - protected final List uploadBufferList; + protected final List bufferList; /** * total remaining of bytebuffer list @@ -65,13 +66,13 @@ public class TieredFileSegmentInputStream extends InputStream { private int markReadPosInCurBuffer = -1; - public TieredFileSegmentInputStream(FileSegmentType fileType, List uploadBufferList, - int contentLength) { + public FileSegmentInputStream( + FileSegmentType fileType, List bufferList, int contentLength) { this.fileType = fileType; this.contentLength = contentLength; - this.uploadBufferList = uploadBufferList; - if (uploadBufferList != null && uploadBufferList.size() > 0) { - this.curBuffer = uploadBufferList.get(curReadBufferIndex); + this.bufferList = bufferList; + if (bufferList != null && bufferList.size() > 0) { + this.curBuffer = bufferList.get(curReadBufferIndex); } } @@ -95,18 +96,34 @@ public synchronized void reset() throws IOException { this.readPosition = markReadPosition; this.curReadBufferIndex = markCurReadBufferIndex; this.readPosInCurBuffer = markReadPosInCurBuffer; - if (this.curReadBufferIndex < uploadBufferList.size()) { - this.curBuffer = uploadBufferList.get(curReadBufferIndex); + if (this.curReadBufferIndex < bufferList.size()) { + this.curBuffer = bufferList.get(curReadBufferIndex); } } + public synchronized void rewind() { + this.readPosition = 0; + this.curReadBufferIndex = 0; + this.readPosInCurBuffer = 0; + if (CollectionUtils.isNotEmpty(bufferList)) { + this.curBuffer = bufferList.get(0); + for (ByteBuffer buffer : bufferList) { + buffer.rewind(); + } + } + } + + public int getContentLength() { + return contentLength; + } + @Override public int available() { return contentLength - readPosition; } - public List getUploadBufferList() { - return uploadBufferList; + public List getBufferList() { + return bufferList; } public ByteBuffer getCodaBuffer() { @@ -121,10 +138,10 @@ public int read() { readPosition++; if (readPosInCurBuffer >= curBuffer.remaining()) { curReadBufferIndex++; - if (curReadBufferIndex >= uploadBufferList.size()) { + if (curReadBufferIndex >= bufferList.size()) { return -1; } - curBuffer = uploadBufferList.get(curReadBufferIndex); + curBuffer = bufferList.get(curReadBufferIndex); readPosInCurBuffer = 0; } return curBuffer.get(readPosInCurBuffer++) & 0xff; @@ -153,8 +170,8 @@ public int read(byte[] b, int off, int len) { int bufIndex = curReadBufferIndex; int posInCurBuffer = readPosInCurBuffer; ByteBuffer curBuf = curBuffer; - while (needRead > 0 && bufIndex < uploadBufferList.size()) { - curBuf = uploadBufferList.get(bufIndex); + while (needRead > 0 && bufIndex < bufferList.size()) { + curBuf = bufferList.get(bufIndex); int remaining = curBuf.remaining() - posInCurBuffer; int readLen = Math.min(remaining, needRead); // read from curBuf diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java similarity index 54% rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java index d0c983fd432..a90baff3ae6 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java @@ -15,30 +15,34 @@ * limitations under the License. */ -package org.apache.rocketmq.tieredstore.provider.inputstream; +package org.apache.rocketmq.tieredstore.provider.stream; import java.nio.ByteBuffer; import java.util.List; import org.apache.rocketmq.tieredstore.common.FileSegmentType; -public class TieredFileSegmentInputStreamFactory { +public class FileSegmentInputStreamFactory { - public static TieredFileSegmentInputStream build(FileSegmentType fileType, - long startOffset, List uploadBufferList, ByteBuffer codaBuffer, int contentLength) { + public static FileSegmentInputStream build( + FileSegmentType fileType, long offset, List bufferList, ByteBuffer byteBuffer, int length) { + + if (bufferList == null) { + throw new IllegalArgumentException("bufferList is null"); + } switch (fileType) { case COMMIT_LOG: - return new TieredCommitLogInputStream( - fileType, startOffset, uploadBufferList, codaBuffer, contentLength); + return new CommitLogInputStream( + fileType, offset, bufferList, byteBuffer, length); case CONSUME_QUEUE: - return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength); + return new FileSegmentInputStream(fileType, bufferList, length); case INDEX: - if (uploadBufferList.size() != 1) { - throw new IllegalArgumentException("uploadBufferList size in INDEX type input stream must be 1"); + if (bufferList.size() != 1) { + throw new IllegalArgumentException("buffer block size must be 1 when file type is IndexFile"); } - return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength); + return new FileSegmentInputStream(fileType, bufferList, length); default: - throw new IllegalArgumentException("fileType is not supported"); + throw new IllegalArgumentException("file type is not supported"); } } } diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java index 8601392e747..07af1fc8b1f 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java @@ -130,36 +130,36 @@ public void testViaTieredStorage() { // TieredStorageLevel.DISABLE properties.setProperty("tieredStorageLevel", "0"); configuration.update(properties); - Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); + Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); // TieredStorageLevel.NOT_IN_DISK properties.setProperty("tieredStorageLevel", "1"); configuration.update(properties); when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false); - Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); + Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); - Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); + Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); // TieredStorageLevel.NOT_IN_MEM properties.setProperty("tieredStorageLevel", "2"); configuration.update(properties); Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false); Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true); - Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); + Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(false); - Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); + Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true); - Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); + Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); // TieredStorageLevel.FORCE properties.setProperty("tieredStorageLevel", "3"); configuration.update(properties); - Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); + Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); } @Test @@ -168,7 +168,7 @@ public void testGetMessageAsync() { GetMessageResult result1 = new GetMessageResult(); result1.setStatus(GetMessageStatus.FOUND); GetMessageResult result2 = new GetMessageResult(); - result2.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING); + result2.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY); when(fetcher.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(result1)); when(nextStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(result2); @@ -188,7 +188,8 @@ public void testGetMessageAsync() { properties.setProperty("tieredStorageLevel", "3"); configuration.update(properties); when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); - Assert.assertSame(result2, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null)); + Assert.assertEquals(result2.getStatus(), + store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null).getStatus()); } @Test diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java index cc39cfbfce3..7e2fbf20136 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java @@ -16,14 +16,12 @@ */ package org.apache.rocketmq.tieredstore.file; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; +import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; +import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor; import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata; import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore; import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; @@ -34,6 +32,11 @@ import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + public class TieredFlatFileTest { private final String storePath = TieredStoreTestUtil.getRandomStorePath(); @@ -55,6 +58,7 @@ public void setUp() throws ClassNotFoundException, NoSuchMethodException { public void tearDown() throws IOException { TieredStoreTestUtil.destroyMetadataStore(); TieredStoreTestUtil.destroyTempDir(storePath); + TieredStoreExecutor.shutdown(); } private List getSegmentMetadataList(TieredMetadataStore metadataStore) { @@ -299,4 +303,40 @@ public void testRollingNewFile() { fileQueue.rollingNewFile(); Assert.assertEquals(2, fileQueue.getFileSegmentCount()); } + + @Test + public void testGetFileByTime() { + String filePath = TieredStoreUtil.toPath(queue); + TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath); + TieredFileSegment fileSegment1 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig); + fileSegment1.setMinTimestamp(100); + fileSegment1.setMaxTimestamp(200); + + TieredFileSegment fileSegment2 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig); + fileSegment2.setMinTimestamp(200); + fileSegment2.setMaxTimestamp(300); + + tieredFlatFile.getFileSegmentList().add(fileSegment1); + tieredFlatFile.getFileSegmentList().add(fileSegment2); + + TieredFileSegment segmentUpper = tieredFlatFile.getFileByTime(400, BoundaryType.UPPER); + Assert.assertEquals(fileSegment2, segmentUpper); + + TieredFileSegment segmentLower = tieredFlatFile.getFileByTime(400, BoundaryType.LOWER); + Assert.assertEquals(fileSegment2, segmentLower); + + + TieredFileSegment segmentUpper2 = tieredFlatFile.getFileByTime(0, BoundaryType.UPPER); + Assert.assertEquals(fileSegment1, segmentUpper2); + + TieredFileSegment segmentLower2 = tieredFlatFile.getFileByTime(0, BoundaryType.LOWER); + Assert.assertEquals(fileSegment1, segmentLower2); + + + TieredFileSegment segmentUpper3 = tieredFlatFile.getFileByTime(200, BoundaryType.UPPER); + Assert.assertEquals(fileSegment1, segmentUpper3); + + TieredFileSegment segmentLower3 = tieredFlatFile.getFileByTime(200, BoundaryType.LOWER); + Assert.assertEquals(fileSegment2, segmentLower3); + } } diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java index 7ef49578dd4..2da72bc7a70 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java @@ -19,9 +19,8 @@ import com.sun.jna.Platform; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; @@ -31,9 +30,7 @@ import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; -import org.junit.Assume; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; public class TieredIndexFileTest { @@ -45,11 +42,12 @@ public class TieredIndexFileTest { @Before public void setUp() { storeConfig = new TieredMessageStoreConfig(); + storeConfig.setBrokerName("IndexFileBroker"); storeConfig.setStorePathRootDir(storePath); - storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment"); - storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2); - storeConfig.setTieredStoreIndexFileMaxIndexNum(3); - mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1); + storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment"); + storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5); + storeConfig.setTieredStoreIndexFileMaxIndexNum(20); + mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1); TieredStoreUtil.getMetadataStore(storeConfig); TieredStoreExecutor.init(); } @@ -61,77 +59,35 @@ public void tearDown() throws IOException { TieredStoreExecutor.shutdown(); } - @Ignore @Test public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException { if (Platform.isWindows()) { return; } - // skip this test on windows - Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); - TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig); TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath); + indexFile.append(mq, 0, "key3", 3, 300, 1000); indexFile.append(mq, 0, "key2", 2, 200, 1100); indexFile.append(mq, 0, "key1", 1, 100, 1200); - Awaitility.waitAtMost(5, TimeUnit.SECONDS) - .until(() -> { - List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); - if (indexList.size() != 1) { - return false; - } - - ByteBuffer indexBuffer = indexList.get(0).getValue(); - Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining()); - - Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4)); - Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8)); - Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); - - Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4)); - Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8)); - Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4)); - return true; - }); - - indexFile.append(mq, 0, "key4", 4, 400, 1300); - indexFile.append(mq, 0, "key4", 4, 400, 1300); - indexFile.append(mq, 0, "key4", 4, 400, 1300); - - Awaitility.waitAtMost(5, TimeUnit.SECONDS) - .until(() -> { - List> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join(); - if (indexList.size() != 1) { - return false; - } - - ByteBuffer indexBuffer = indexList.get(0).getValue(); - Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); - Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); - Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); - Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); - return true; - }); - - List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join(); + // do not do schedule task here + TieredStoreExecutor.shutdown(); + List> indexList = + indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); Assert.assertEquals(0, indexList.size()); - indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join(); - Assert.assertEquals(2, indexList.size()); + // do compaction once + TieredStoreExecutor.init(); + storeConfig.setTieredStoreIndexFileRollingIdleInterval(0); + indexFile.doScheduleTask(); + Awaitility.await().atMost(Duration.ofSeconds(10)) + .until(() -> !indexFile.getPreMappedFile().getFile().exists()); - ByteBuffer indexBuffer = indexList.get(0).getValue(); - Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); - Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); - Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); - Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); + indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); + Assert.assertEquals(1, indexList.size()); - indexBuffer = indexList.get(1).getValue(); - Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining()); - Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4)); - Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8)); - Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); + indexFile.destroy(); } } diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java similarity index 82% rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java index a6566b7de5a..3bbe41dd4b6 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java @@ -20,13 +20,13 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.util.List; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; -public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStream { +public class MockFileSegmentInputStream extends FileSegmentInputStream { private final InputStream inputStream; - public MockTieredFileSegmentInputStream(InputStream inputStream) { + public MockFileSegmentInputStream(InputStream inputStream) { super(null, null, Integer.MAX_VALUE); this.inputStream = inputStream; } @@ -43,7 +43,7 @@ public int read() { } @Override - public List getUploadBufferList() { + public List getBufferList() { return null; } diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java index a2554ba3d13..743d9182ce3 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java @@ -28,8 +28,8 @@ import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.file.TieredCommitLog; import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory; import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest; import org.junit.Assert; @@ -57,7 +57,7 @@ public void testCommitLogTypeInputStream() { bufferSize += byteBuffer.remaining(); } - // build expected byte buffer for verifying the TieredFileSegmentInputStream + // build expected byte buffer for verifying the FileSegmentInputStream ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); for (ByteBuffer byteBuffer : uploadBufferList) { expectedByteBuffer.put(byteBuffer); @@ -74,7 +74,7 @@ public void testCommitLogTypeInputStream() { int[] batchReadSizeTestSet = { MessageBufferUtil.PHYSICAL_OFFSET_POSITION - 1, MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtil.PHYSICAL_OFFSET_POSITION + 1, MSG_LEN - 1, MSG_LEN, MSG_LEN + 1 }; - verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( + verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), finalBufferSize, batchReadSizeTestSet); } @@ -98,7 +98,7 @@ public void testCommitLogTypeInputStreamWithCoda() { int codaBufferSize = codaBuffer.remaining(); bufferSize += codaBufferSize; - // build expected byte buffer for verifying the TieredFileSegmentInputStream + // build expected byte buffer for verifying the FileSegmentInputStream ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); for (ByteBuffer byteBuffer : uploadBufferList) { expectedByteBuffer.put(byteBuffer); @@ -119,7 +119,7 @@ public void testCommitLogTypeInputStreamWithCoda() { MSG_LEN - 1, MSG_LEN, MSG_LEN + 1, bufferSize - 1, bufferSize, bufferSize + 1 }; - verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( + verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, codaBuffer, finalBufferSize), finalBufferSize, batchReadSizeTestSet); } @@ -134,7 +134,7 @@ public void testConsumeQueueTypeInputStream() { bufferSize += byteBuffer.remaining(); } - // build expected byte buffer for verifying the TieredFileSegmentInputStream + // build expected byte buffer for verifying the FileSegmentInputStream ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); for (ByteBuffer byteBuffer : uploadBufferList) { expectedByteBuffer.put(byteBuffer); @@ -143,7 +143,7 @@ public void testConsumeQueueTypeInputStream() { int finalBufferSize = bufferSize; int[] batchReadSizeTestSet = {TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE - 1, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE + 1}; - verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( + verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( FileSegmentType.CONSUME_QUEUE, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), bufferSize, batchReadSizeTestSet); } @@ -156,16 +156,16 @@ public void testIndexTypeInputStream() { byteBuffer.flip(); List uploadBufferList = Arrays.asList(byteBuffer); - // build expected byte buffer for verifying the TieredFileSegmentInputStream + // build expected byte buffer for verifying the FileSegmentInputStream ByteBuffer expectedByteBuffer = byteBuffer.slice(); - verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( + verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( FileSegmentType.INDEX, COMMIT_LOG_START_OFFSET, uploadBufferList, null, byteBuffer.limit()), byteBuffer.limit(), new int[] {23, 24, 25}); } - private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier constructor, + private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier constructor, int bufferSize, int[] readBatchSizeTestSet) { - TieredFileSegmentInputStream inputStream = constructor.get(); + FileSegmentInputStream inputStream = constructor.get(); // verify verifyInputStream(inputStream, expectedByteBuffer); diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java index 4cd83e0d260..a655710a500 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java @@ -116,13 +116,22 @@ public void testConsumeQueue() { } @Test - public void testCommitFailed() { + public void testCommitFailedThenSuccess() { long startTime = System.currentTimeMillis(); MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG); long lastSize = segment.getSize(); - segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0); - segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0); - + segment.setCheckSize(false); + segment.initPosition(lastSize); + segment.setSize((int) lastSize); + + ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( + MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize); + ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( + MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN); + segment.append(buffer1, 0); + segment.append(buffer2, 0); + + // Mock new message arrive segment.blocker = new CompletableFuture<>(); new Thread(() -> { try { @@ -131,20 +140,88 @@ public void testCommitFailed() { Assert.fail(e.getMessage()); } ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer(); + buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2); buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime); segment.append(buffer, 0); segment.blocker.complete(false); }).start(); + // Commit failed segment.commit(); segment.blocker.join(); + segment.blocker = null; + + // Copy data and assume commit success + segment.getMemStore().put(buffer1); + segment.getMemStore().put(buffer2); + segment.setSize((int) (lastSize + MessageBufferUtilTest.MSG_LEN * 2)); - segment.blocker = new CompletableFuture<>(); - segment.blocker.complete(true); segment.commit(); + Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition()); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset()); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); + + ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN); + Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1)); + + ByteBuffer msg2 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtil.getCommitLogOffset(msg2)); + + ByteBuffer msg3 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtil.getCommitLogOffset(msg3)); + } + + @Test + public void testCommitFailed3Times() { + long startTime = System.currentTimeMillis(); + MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG); + long lastSize = segment.getSize(); + segment.setCheckSize(false); + segment.initPosition(lastSize); + segment.setSize((int) lastSize); + + ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( + MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize); + ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( + MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN); + segment.append(buffer1, 0); + segment.append(buffer2, 0); + + // Mock new message arrive + segment.blocker = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + Assert.fail(e.getMessage()); + } + ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer(); + buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2); + buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime); + segment.append(buffer, 0); + segment.blocker.complete(false); + }).start(); + + for (int i = 0; i < 3; i++) { + segment.commit(); + } + + Assert.assertEquals(lastSize, segment.getCommitPosition()); + Assert.assertEquals(baseOffset + lastSize, segment.getCommitOffset()); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); + + segment.blocker.join(); + segment.blocker = null; + segment.commit(); + Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitPosition()); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitOffset()); Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); + + segment.commit(); + Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition()); Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset()); + Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN); Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1)); diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java new file mode 100644 index 00000000000..fbaafa1b4cf --- /dev/null +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tieredstore.provider; + +import org.apache.rocketmq.common.topic.TopicValidator; +import org.junit.Assert; +import org.junit.Test; + +public class TieredStoreTopicBlackListFilterTest { + + @Test + public void filterTopicTest() { + TieredStoreTopicFilter topicFilter = new TieredStoreTopicBlackListFilter(); + Assert.assertTrue(topicFilter.filterTopic("")); + Assert.assertTrue(topicFilter.filterTopic(TopicValidator.SYSTEM_TOPIC_PREFIX + "_Topic")); + + String topicName = "WhiteTopic"; + Assert.assertFalse(topicFilter.filterTopic(topicName)); + topicFilter.addTopicToBlackList(topicName); + Assert.assertTrue(topicFilter.filterTopic(topicName)); + } +} \ No newline at end of file diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java index cb155cf8f70..80ad41f6859 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java @@ -23,7 +23,7 @@ import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; import org.junit.Assert; @@ -33,6 +33,8 @@ public class MemoryFileSegment extends TieredFileSegment { public CompletableFuture blocker; + protected int size = 0; + protected boolean checkSize = true; public MemoryFileSegment(FileSegmentType fileType, MessageQueue messageQueue, long baseOffset, @@ -56,6 +58,18 @@ public MemoryFileSegment(TieredMessageStoreConfig storeConfig, memStore.position((int) getSize()); } + public boolean isCheckSize() { + return checkSize; + } + + public void setCheckSize(boolean checkSize) { + this.checkSize = checkSize; + } + + public ByteBuffer getMemStore() { + return memStore; + } + @Override public String getPath() { return filePath; @@ -66,7 +80,11 @@ public long getSize() { if (checkSize) { return 1000; } - return 0; + return size; + } + + public void setSize(int size) { + this.size = size; } @Override @@ -85,11 +103,11 @@ public CompletableFuture read0(long position, int length) { @Override public CompletableFuture commit0( - TieredFileSegmentInputStream inputStream, long position, int length, boolean append) { + FileSegmentInputStream inputStream, long position, int length, boolean append) { try { if (blocker != null && !blocker.get()) { - throw new IllegalStateException(); + throw new IllegalStateException("Commit Exception for Memory Test"); } } catch (InterruptedException | ExecutionException e) { Assert.fail(e.getMessage()); @@ -98,7 +116,6 @@ public CompletableFuture commit0( Assert.assertTrue(!checkSize || position >= getSize()); byte[] buffer = new byte[1024]; - int startPos = memStore.position(); try { int len; diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java index 8ac330b3700..630fd22236c 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java @@ -22,7 +22,7 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; -import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; import org.junit.Assert; @@ -46,7 +46,7 @@ public long getSize() { } @Override - public CompletableFuture commit0(TieredFileSegmentInputStream inputStream, long position, int length, + public CompletableFuture commit0(FileSegmentInputStream inputStream, long position, int length, boolean append) { try { if (blocker != null && !blocker.get()) { diff --git a/tools/pom.xml b/tools/pom.xml index 1c3b431bc20..e1daa57a62f 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.1.4-SNAPSHOT + 5.1.5-SNAPSHOT 4.0.0 diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index fa3596d51c3..1ebff6d8afc 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -66,6 +66,7 @@ import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.NetworkUtil; import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.RPCHook; @@ -193,7 +194,7 @@ public void start() throws MQClientException { int threadPoolCoreSize = Integer.parseInt(System.getProperty("rocketmq.admin.threadpool.coresize", "20")); - this.threadPoolExecutor = new ThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_")); + this.threadPoolExecutor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_")); break; case RUNNING: diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index 0c2618e91c2..788fa83c2cd 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; import org.apache.rocketmq.tools.command.message.SendMessageCommand; +import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand; import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; @@ -255,6 +256,7 @@ public static void initCommand() { initCommand(new ExportMetadataCommand()); initCommand(new ExportConfigsCommand()); initCommand(new ExportMetricsCommand()); + initCommand(new ExportMetadataInRocksDBCommand()); initCommand(new HAStatusSubCommand()); @@ -278,7 +280,7 @@ private static void printHelp() { System.out.printf("The most commonly used mqadmin commands are:%n"); for (SubCommand cmd : SUB_COMMANDS) { - System.out.printf(" %-25s %s%n", cmd.commandName(), cmd.commandDesc()); + System.out.printf(" %-35s %s%n", cmd.commandName(), cmd.commandDesc()); } System.out.printf("%nSee 'mqadmin help ' for more information on a specific command.%n"); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java index f8a00b1e0bb..26ed028fb33 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java @@ -47,7 +47,7 @@ public String commandName() { @Override public String commandDesc() { - return "List all of acl config version information in cluster"; + return "List all of acl config version information in cluster."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java index fd3a92fffa9..a7f3d295a72 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java @@ -42,7 +42,7 @@ public String commandAlias() { @Override public String commandDesc() { - return "Delete Acl Config Account in broker"; + return "Delete Acl Config Account in broker."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java index 25844d6a194..f1c9a14969f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java @@ -49,7 +49,7 @@ public String commandAlias() { @Override public String commandDesc() { - return "List all of acl config information in cluster"; + return "List all of acl config information in cluster."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java index 3be40daa17a..d8a06f92d14 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java @@ -40,7 +40,7 @@ public String commandName() { @Override public String commandDesc() { - return "Update acl config yaml file in broker"; + return "Update acl config yaml file in broker."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java index ff662b50694..9dacf1fae64 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java @@ -37,7 +37,7 @@ public String commandName() { @Override public String commandDesc() { - return "Update global white address for acl Config File in broker"; + return "Update global white address for acl Config File in broker."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java index 3f2f9067309..7658a2139c5 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java @@ -61,7 +61,7 @@ public String commandName() { @Override public String commandDesc() { - return "Fetch broker consume stats data"; + return "Fetch broker consume stats data."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java index 830ff3425e2..ce934f547b6 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java @@ -44,7 +44,7 @@ public String commandName() { @Override public String commandDesc() { - return "Fetch broker runtime status data"; + return "Fetch broker runtime status data."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java index b00c7f5f580..4fdabfdf858 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java @@ -44,7 +44,7 @@ public String commandName() { @Override public String commandDesc() { - return "set read ahead mode for all commitlog files"; + return "Set read ahead mode for all commitlog files."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java index a4b2a51adbd..142bb7b3c68 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java @@ -37,7 +37,7 @@ public String commandName() { @Override public String commandDesc() { - return "Delete expired CommitLog files"; + return "Delete expired CommitLog files."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java index 5d86c10e455..c4762a29606 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java @@ -45,7 +45,7 @@ public String commandName() { @Override public String commandDesc() { - return "Get broker config by cluster or special broker"; + return "Get broker config by cluster or special broker."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java index abe8fc62215..1a8961e046d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java @@ -38,7 +38,7 @@ public String commandName() { @Override public String commandDesc() { - return "Fetch broker epoch entries"; + return "Fetch broker epoch entries."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java index 7c54e650c37..34b3ba7d306 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java @@ -47,7 +47,7 @@ public String commandName() { @Override public String commandDesc() { - return "get cold data flow ctr info"; + return "Get cold data flow ctr info."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java index b0477924f27..f2040748033 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java @@ -36,7 +36,7 @@ public String commandName() { @Override public String commandDesc() { - return "remove consumer from cold ctr config"; + return "Remove consumer from cold ctr config."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java index b2ac48c84c6..90451b51f5d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java @@ -33,7 +33,7 @@ public String commandName() { @Override public String commandDesc() { - return "Reset master flush offset in slave"; + return "Reset master flush offset in slave."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java index 98abeb6aede..62816ef03a7 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java @@ -37,7 +37,7 @@ public String commandName() { @Override public String commandDesc() { - return "Update broker's config"; + return "Update broker's config."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java index d06a24b5794..8d1a0007770 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java @@ -39,7 +39,7 @@ public String commandName() { @Override public String commandDesc() { - return "addOrUpdate cold data flow ctr group config"; + return "Add or update cold data flow ctr group config."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java index 7253970bd20..d755e9e5d83 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java @@ -48,7 +48,7 @@ public String commandName() { @Override public String commandDesc() { - return "List All clusters Message Send RT"; + return "List All clusters Message Send RT."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java index a7a840a443f..ede0fa5cf4e 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java @@ -41,7 +41,7 @@ public String commandName() { @Override public String commandDesc() { - return "List cluster infos"; + return "List cluster infos."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java index 630961e31b8..35f73d8a03a 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java @@ -39,7 +39,7 @@ public String commandName() { @Override public String commandDesc() { - return "Query consumer's socket connection, client version and subscription"; + return "Query consumer's socket connection, client version and subscription."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java index 2533982c8fc..bde674ab29b 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java @@ -36,7 +36,7 @@ public String commandName() { @Override public String commandDesc() { - return "Query producer's socket connection and client version"; + return "Query producer's socket connection and client version."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java index 72b9c975e20..d8f6f9aa929 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java @@ -47,7 +47,7 @@ public String commandName() { @Override public String commandDesc() { - return "Query consumer's internal data structure"; + return "Query consumer's internal data structure."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java index 6095e766859..4a8253a0254 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java @@ -43,7 +43,7 @@ public String commandName() { @Override public String commandDesc() { - return "Get consumer config by subscription group name"; + return "Get consumer config by subscription group name."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java index 2d08d0bd034..f5e140433e3 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java @@ -34,7 +34,7 @@ public String commandName() { @Override public String commandDesc() { - return "Start Monitoring"; + return "Start Monitoring."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java index f87bafc9303..b17da4de45f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java @@ -41,7 +41,7 @@ public String commandName() { @Override public String commandDesc() { - return "Update or create subscription group"; + return "Update or create subscription group."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java index e9e5be4a593..007d42ae6dc 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java @@ -33,7 +33,7 @@ public String commandName() { @Override public String commandDesc() { - return "Add a broker to specified container"; + return "Add a broker to specified container."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java index 7c455f85846..ab25d8ebed0 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java @@ -33,7 +33,7 @@ public String commandName() { @Override public String commandDesc() { - return "Remove a broker from specified container"; + return "Remove a broker from specified container."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java index 856e4b4269d..24ed025665a 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java @@ -37,7 +37,7 @@ public String commandName() { @Override public String commandDesc() { - return "Clean metadata of broker on controller"; + return "Clean metadata of broker on controller."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java index 70bd7f8e985..96644312708 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java @@ -34,7 +34,7 @@ public String commandName() { @Override public String commandDesc() { - return "Get controller cluster's metadata"; + return "Get controller cluster's metadata."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java index 1affe81f9c1..a522a903df8 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java @@ -37,7 +37,7 @@ public String commandName() { @Override public String commandDesc() { - return "Re-elect the specified broker as master"; + return "Re-elect the specified broker as master."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java index b8191296d9d..c3f96d59723 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Arrays; import java.util.Properties; import com.alibaba.fastjson.JSON; @@ -42,7 +43,7 @@ public String commandName() { @Override public String commandDesc() { - return "Export configs"; + return "Export configs."; } @Override @@ -106,24 +107,33 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) } } + private Properties needBrokerProprties(Properties properties) { + List propertyKeys = Arrays.asList( + "brokerClusterName", + "brokerId", + "brokerName", + "brokerRole", + "fileReservedTime", + "filterServerNums", + "flushDiskType", + "maxMessageSize", + "messageDelayLevel", + "msgTraceTopicName", + "slaveReadEnable", + "traceOn", + "traceTopicEnable", + "useTLS", + "autoCreateTopicEnable", + "autoCreateSubscriptionGroup" + ); + Properties newProperties = new Properties(); - newProperties.setProperty("brokerClusterName", properties.getProperty("brokerClusterName")); - newProperties.setProperty("brokerId", properties.getProperty("brokerId")); - newProperties.setProperty("brokerName", properties.getProperty("brokerName")); - newProperties.setProperty("brokerRole", properties.getProperty("brokerRole")); - newProperties.setProperty("fileReservedTime", properties.getProperty("fileReservedTime")); - newProperties.setProperty("filterServerNums", properties.getProperty("filterServerNums")); - newProperties.setProperty("flushDiskType", properties.getProperty("flushDiskType")); - newProperties.setProperty("maxMessageSize", properties.getProperty("maxMessageSize")); - newProperties.setProperty("messageDelayLevel", properties.getProperty("messageDelayLevel")); - newProperties.setProperty("msgTraceTopicName", properties.getProperty("msgTraceTopicName")); - newProperties.setProperty("slaveReadEnable", properties.getProperty("slaveReadEnable")); - newProperties.setProperty("traceOn", properties.getProperty("traceOn")); - newProperties.setProperty("traceTopicEnable", properties.getProperty("traceTopicEnable")); - newProperties.setProperty("useTLS", properties.getProperty("useTLS")); - newProperties.setProperty("autoCreateTopicEnable", properties.getProperty("autoCreateTopicEnable")); - newProperties.setProperty("autoCreateSubscriptionGroup", properties.getProperty("autoCreateSubscriptionGroup")); + propertyKeys.stream() + .filter(key -> properties.getProperty(key) != null) + .forEach(key -> newProperties.setProperty(key, properties.getProperty(key))); + return newProperties; } + } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java index 1f9cf7d962e..748f7b16e12 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java @@ -46,7 +46,7 @@ public String commandName() { @Override public String commandDesc() { - return "Export metadata"; + return "Export metadata."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java new file mode 100644 index 00000000000..1ecb1fa2cd9 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.export; + +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.config.ConfigRocksDBStorage; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.rocksdb.RocksIterator; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; + +public class ExportMetadataInRocksDBCommand implements SubCommand { + private static final String TOPICS_JSON_CONFIG = "topics"; + private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; + + @Override + public String commandName() { + return "exportMetadataInRocksDB"; + } + + @Override + public String commandDesc() { + return "export RocksDB kv config (topics/subscriptionGroups)"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option pathOption = new Option("p", "path", true, + "Absolute path for the metadata directory"); + pathOption.setRequired(true); + options.addOption(pathOption); + + Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + + "topics/subscriptionGroups"); + configTypeOption.setRequired(true); + options.addOption(configTypeOption); + + Option jsonEnableOption = new Option("j", "jsonEnable", true, + "Json format enable, Default: false"); + jsonEnableOption.setRequired(false); + options.addOption(jsonEnableOption); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { + String path = commandLine.getOptionValue("path").trim(); + if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { + System.out.print("RocksDB path is invalid.\n"); + return; + } + + String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + + boolean jsonEnable = false; + if (commandLine.hasOption("jsonEnable")) { + jsonEnable = Boolean.parseBoolean(commandLine.getOptionValue("jsonEnable").trim()); + } + + + ConfigRocksDBStorage kvStore = new ConfigRocksDBStorage(path, true /* readOnly */); + if (!kvStore.start()) { + System.out.print("RocksDB load error, path=" + path + "\n"); + return; + } + + try { + if (TOPICS_JSON_CONFIG.equalsIgnoreCase(configType) || SUBSCRIPTION_GROUP_JSON_CONFIG.equalsIgnoreCase(configType)) { + handleExportMetadata(kvStore, configType, jsonEnable); + } else { + System.out.printf("Invalid config type=%s, Options: topics,subscriptionGroups\n", configType); + } + } finally { + kvStore.shutdown(); + } + } + + private static void handleExportMetadata(ConfigRocksDBStorage kvStore, String configType, boolean jsonEnable) { + if (jsonEnable) { + final Map jsonConfig = new HashMap<>(); + final Map configTable = new HashMap<>(); + iterateKvStore(kvStore, (key, value) -> { + final String configKey = new String(key, DataConverter.CHARSET_UTF8); + final String configValue = new String(value, DataConverter.CHARSET_UTF8); + final JSONObject jsonObject = JSONObject.parseObject(configValue); + configTable.put(configKey, jsonObject); + } + ); + + jsonConfig.put(configType.equalsIgnoreCase(TOPICS_JSON_CONFIG) ? "topicConfigTable" : "subscriptionGroupTable", + (JSONObject) JSONObject.toJSON(configTable)); + final String jsonConfigStr = JSONObject.toJSONString(jsonConfig, true); + System.out.print(jsonConfigStr + "\n"); + } else { + AtomicLong count = new AtomicLong(0); + iterateKvStore(kvStore, (key, value) -> { + final String configKey = new String(key, DataConverter.CHARSET_UTF8); + final String configValue = new String(value, DataConverter.CHARSET_UTF8); + System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue); + }); + } + } + + private static void iterateKvStore(ConfigRocksDBStorage kvStore, BiConsumer biConsumer) { + try (RocksIterator iterator = kvStore.iterator()) { + iterator.seekToFirst(); + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + biConsumer.accept(iterator.key(), iterator.value()); + } + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java index a793b4b84bf..5d8bb37ba0f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java @@ -56,7 +56,7 @@ public String commandName() { @Override public String commandDesc() { - return "Export metrics"; + return "Export metrics."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java index 44b3ec3e18b..b6231e4f9b1 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java @@ -40,7 +40,7 @@ public String commandName() { @Override public String commandDesc() { - return "Fetch syncStateSet for target brokers"; + return "Fetch syncStateSet for target brokers."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java index b1795e046f4..931658a08fc 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java @@ -41,7 +41,7 @@ public String commandName() { @Override public String commandDesc() { - return "Fetch ha runtime status data"; + return "Fetch ha runtime status data."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java index 4c6d5ffb604..b15b59d505e 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java @@ -40,7 +40,7 @@ public String commandName() { @Override public String commandDesc() { - return "Check message send response time"; + return "Check message send response time."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java index 8aed59ea449..02ff53269f6 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java @@ -70,7 +70,7 @@ public String commandName() { @Override public String commandDesc() { - return "Consume message"; + return "Consume message."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java index ae6d9bdcf21..eee8f3d4bde 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java @@ -38,7 +38,7 @@ public class DumpCompactionLogCommand implements SubCommand { @Override public String commandDesc() { - return "parse compaction log to message"; + return "Parse compaction log to message."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java index 654560167fd..0418e88a706 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java @@ -108,7 +108,7 @@ public String commandName() { @Override public String commandDesc() { - return "Print Message Detail"; + return "Print Message Detail by queueId."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java index d01c36d425d..bb82f5079e5 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java @@ -62,7 +62,7 @@ public String commandName() { @Override public String commandDesc() { - return "Print Message Detail"; + return "Print Message Detail."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java index 2880477f1f4..b42612150a3 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java @@ -186,7 +186,7 @@ public String commandName() { @Override public String commandDesc() { - return "Query Message by Id"; + return "Query Message by Id."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java index ba7b00c3bbe..64627fd19fa 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java @@ -36,7 +36,7 @@ public String commandName() { @Override public String commandDesc() { - return "Query Message by Key"; + return "Query Message by Key."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java index d27313af1e2..14d0625fd2c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java @@ -39,7 +39,7 @@ public String commandName() { @Override public String commandDesc() { - return "Query Message by offset"; + return "Query Message by offset."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java index 1b28f8be102..b71cee90160 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java @@ -141,7 +141,7 @@ public String commandName() { @Override public String commandDesc() { - return "Query Message by Unique key"; + return "Query Message by Unique key."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java index 2b982efefdd..2c546ec563e 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java @@ -65,7 +65,7 @@ public Options buildCommandlineOptions(Options options) { @Override public String commandDesc() { - return "Query a message trace"; + return "Query a message trace."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java index 836ee192b2d..970da6b1690 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java @@ -41,7 +41,7 @@ public String commandName() { @Override public String commandDesc() { - return "Send a message"; + return "Send a message."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java new file mode 100644 index 00000000000..b987ad873be --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.metadata; + +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.config.RocksDBConfigManager; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +public class RocksDBConfigToJsonCommand implements SubCommand { + private static final String TOPICS_JSON_CONFIG = "topics"; + private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; + + @Override + public String commandName() { + return "rocksDBConfigToJson"; + } + + @Override + public String commandDesc() { + return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option pathOption = new Option("p", "path", true, + "Absolute path to the metadata directory"); + pathOption.setRequired(true); + options.addOption(pathOption); + + Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + + "topics/subscriptionGroups"); + configTypeOption.setRequired(true); + options.addOption(configTypeOption); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { + String path = commandLine.getOptionValue("path").trim(); + if (StringUtils.isEmpty(path) || !new File(path).exists()) { + System.out.print("Rocksdb path is invalid.\n"); + return; + } + + String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + + final long memTableFlushInterval = 60 * 60 * 1000L; + RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); + try { + if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { + // for topics.json + final Map topicsJsonConfig = new HashMap<>(); + final Map topicConfigTable = new HashMap<>(); + boolean isLoad = kvConfigManager.load(path, (key, value) -> { + final String topic = new String(key, DataConverter.CHARSET_UTF8); + final String topicConfig = new String(value, DataConverter.CHARSET_UTF8); + final JSONObject jsonObject = JSONObject.parseObject(topicConfig); + topicConfigTable.put(topic, jsonObject); + }); + + if (isLoad) { + topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); + final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); + System.out.print(topicsJsonStr + "\n"); + return; + } + } + if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { + // for subscriptionGroup.json + final Map subscriptionGroupJsonConfig = new HashMap<>(); + final Map subscriptionGroupTable = new HashMap<>(); + boolean isLoad = kvConfigManager.load(path, (key, value) -> { + final String subscriptionGroup = new String(key, DataConverter.CHARSET_UTF8); + final String subscriptionGroupConfig = new String(value, DataConverter.CHARSET_UTF8); + final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); + subscriptionGroupTable.put(subscriptionGroup, jsonObject); + }); + + if (isLoad) { + subscriptionGroupJsonConfig.put("subscriptionGroupTable", + (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); + final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); + System.out.print(subscriptionGroupJsonStr + "\n"); + return; + } + } + System.out.print("Config type was not recognized, configType=" + configType + "\n"); + } finally { + kvConfigManager.stop(); + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java index 98542d065d5..0b0a075bd17 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java @@ -34,7 +34,7 @@ public String commandName() { @Override public String commandDesc() { - return "Add write perm of broker in all name server you defined in the -n param"; + return "Add write perm of broker in all name server you defined in the -n param."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java index 213931ed86e..637dd52c861 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java @@ -34,7 +34,7 @@ public String commandName() { @Override public String commandDesc() { - return "Wipe write perm of broker in all name server you defined in the -n param"; + return "Wipe write perm of broker in all name server you defined in the -n param."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java index 139821f9c0d..b22491a5918 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java @@ -41,7 +41,7 @@ public String commandName() { @Override public String commandDesc() { - return "Skip all messages that are accumulated (not consumed) currently"; + return "Skip all messages that are accumulated (not consumed) currently."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java index 1d49bbe1167..96097a93ed3 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java @@ -144,7 +144,7 @@ public String commandName() { @Override public String commandDesc() { - return "Topic and Consumer tps stats"; + return "Topic and Consumer tps stats."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java index 3fa42f297a6..6a9b81eb8a9 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java @@ -41,7 +41,7 @@ public String commandName() { @Override public String commandDesc() { - return "Allocate MQ"; + return "Allocate MQ."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java index 1dab693d950..098f34ff0b0 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java @@ -34,7 +34,7 @@ public String commandName() { @Override public String commandDesc() { - return "Get cluster info for topic"; + return "Get cluster info for topic."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java index 346bac704bb..d9a279f80e3 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java @@ -45,7 +45,7 @@ public String commandName() { @Override public String commandDesc() { - return "Fetch all topic list from name server"; + return "Fetch all topic list from name server."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java index f2dabec4ea2..70949d388cf 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java @@ -42,7 +42,7 @@ public String commandName() { @Override public String commandDesc() { - return "Examine topic route info"; + return "Examine topic route info."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java index fdb249fab67..a1619ecedfd 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java @@ -40,7 +40,7 @@ public String commandName() { @Override public String commandDesc() { - return "Examine topic Status info"; + return "Examine topic Status info."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java index bebc646b43e..3040d04c26d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java @@ -36,7 +36,7 @@ public String commandName() { @Override public String commandDesc() { - return "Create or update or delete order conf"; + return "Create or update or delete order conf."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java index 85a18c654b6..3daeee86cad 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java @@ -48,7 +48,7 @@ public String commandName() { @Override public String commandDesc() { - return "Update or create static topic, which has fixed number of queues"; + return "Update or create static topic, which has fixed number of queues."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java index aaa88153808..d27cd18613d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java @@ -44,7 +44,7 @@ public String commandName() { @Override public String commandDesc() { - return "Update topic perm"; + return "Update topic perm."; } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java index b68463396b4..2989141756a 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java @@ -42,7 +42,7 @@ public String commandName() { @Override public String commandDesc() { - return "Update or create topic"; + return "Update or create topic."; } @Override diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java new file mode 100644 index 00000000000..2b938c90fb8 --- /dev/null +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.metadata; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand; +import org.junit.Test; + +import java.io.File; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ExportMetadataInRocksDBCommandTest { + private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; + + @Test + public void testExecute() throws SubCommandException { + { + String[][] cases = new String[][] { + {"topics", "false"}, + {"topics", "false1"}, + {"topics", "true"}, + {"subscriptionGroups", "false"}, + {"subscriptionGroups", "false2"}, + {"subscriptionGroups", "true"} + }; + + for (String[] c : cases) { + ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[] {"-p " + BASE_PATH + c[0], "-t " + c[0], "-j " + c[1]}; + final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, + cmd.buildCommandlineOptions(options), new DefaultParser()); + cmd.execute(commandLine, options, null); + assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c[0]); + assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c[0]); + assertThat(commandLine.getOptionValue("j").trim()).isEqualTo(c[1]); + } + } + // invalid cases + { + String[][] cases = new String[][] { + {"-p " + BASE_PATH + "tmpPath", "-t topics", "-j true"}, + {"-p ", "-t topics", "-j true"}, + {"-p " + BASE_PATH + "topics", "-t invalid_type", "-j true"} + }; + + for (String[] c : cases) { + ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, + cmd.buildCommandlineOptions(options), new DefaultParser()); + cmd.execute(commandLine, options, null); + } + } + } +}