From 33d0b417cea4f7cef8c2cb0e4a9cc0a6d81f89ca Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 5 Aug 2024 14:54:31 +0800 Subject: [PATCH 1/7] [feat](lock)add deadlock detection tool and monitored lock implementations --- .../common/lock/AbstractMonitoredLock.java | 104 +++++++++++++++ .../doris/common/lock/DeadlockMonitor.java | 102 +++++++++++++++ .../common/lock/MonitoredReentrantLock.java | 83 ++++++++++++ .../lock/MonitoredReentrantReadWriteLock.java | 123 ++++++++++++++++++ 4 files changed, 412 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java new file mode 100644 index 00000000000000..da883b69b1c6bd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.lock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract base class for a monitored lock that tracks lock acquisition, + * release, and attempt times. It provides mechanisms for monitoring the + * duration for which a lock is held and logging any instances where locks + * are held longer than a specified timeout or fail to be acquired within + * a specified timeout. + */ +public abstract class AbstractMonitoredLock { + protected static final long HOLD_TIMEOUT = 5000; // Lock hold timeout in milliseconds + private static final Logger LOG = LoggerFactory.getLogger(AbstractMonitoredLock.class); + + // Thread-local variable to store the lock start time + private final ThreadLocal lockStartTime = new ThreadLocal<>(); + + + /** + * Method to be called after successfully acquiring the lock. + * Sets the start time for the lock. + */ + protected void afterLock() { + lockStartTime.set(System.nanoTime()); + } + + /** + * Method to be called after releasing the lock. + * Calculates the lock hold time and logs a warning if it exceeds the hold timeout. + */ + protected void afterUnlock() { + Long startTime = lockStartTime.get(); + if (startTime != null) { + long lockHoldTimeNanos = System.nanoTime() - startTime; + long lockHoldTimeMs = lockHoldTimeNanos >> 20; + Thread currentThread = Thread.currentThread(); + + if (lockHoldTimeMs > HOLD_TIMEOUT) { + String stackTrace = getThreadStackTrace(currentThread.getStackTrace()); + LOG.warn("Thread ID: {}, Thread Name: {} - Lock held for {} ms, exceeding hold timeout of {} ms \n" + + "Thread stack trace:\n{}", + currentThread.getId(), currentThread.getName(), lockHoldTimeMs, HOLD_TIMEOUT, stackTrace); + } + lockStartTime.remove(); + } + } + + /** + * Method to be called after attempting to acquire the lock using tryLock. + * Logs a warning if the lock was not acquired within a reasonable time. + * + * @param acquired Whether the lock was successfully acquired + * @param startTime The start time of the lock attempt + */ + protected void afterTryLock(boolean acquired, long startTime) { + if (acquired) { + afterLock(); + return; + } + if (LOG.isDebugEnabled()) { + long elapsedTime = (System.nanoTime() - startTime) >> 20; + Thread currentThread = Thread.currentThread(); + String stackTrace = getThreadStackTrace(currentThread.getStackTrace()); + LOG.debug("Thread ID: {}, Thread Name: {} - Failed to acquire the lock within {} ms" + + "\nThread blocking info:\n{}", + currentThread.getId(), currentThread.getName(), elapsedTime, stackTrace); + } + } + + /** + * Utility method to format the stack trace of a thread. + * + * @param stackTrace The stack trace elements of the thread + * @return A formatted string of the stack trace + */ + private String getThreadStackTrace(StackTraceElement[] stackTrace) { + StringBuilder sb = new StringBuilder(); + for (StackTraceElement element : stackTrace) { + sb.append("\tat ").append(element).append("\n"); + } + return sb.toString(); + } +} + + diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java new file mode 100644 index 00000000000000..0c22ad17607f48 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.lock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * A utility class for monitoring and reporting deadlocks in a Java application. + *

+ * This class uses the Java Management API to periodically check for deadlocked threads + * and logs detailed information about any detected deadlocks. It can be configured to + * run at a fixed interval. + *

+ */ +public class DeadlockMonitor { + private static final Logger LOG = LoggerFactory.getLogger(DeadlockMonitor.class); + + private final ThreadMXBean threadMXBean; + private final ScheduledExecutorService scheduler; + + public DeadlockMonitor() { + this.threadMXBean = ManagementFactory.getThreadMXBean(); + this.scheduler = Executors.newScheduledThreadPool(1); + } + + /** + * Starts monitoring for deadlocks at a fixed rate. + * + * @param period the period between successive executions + * @param unit the time unit of the period parameter + */ + public void startMonitoring(long period, TimeUnit unit) { + scheduler.scheduleAtFixedRate(this::detectAndReportDeadlocks, 0, period, unit); + } + + /** + * Detects and reports deadlocks if any are found. + */ + public void detectAndReportDeadlocks() { + // Get IDs of threads that are deadlocked + long[] deadlockedThreadIds = threadMXBean.findDeadlockedThreads(); + + // Check if there are no deadlocked threads + if (deadlockedThreadIds == null || deadlockedThreadIds.length == 0) { + LOG.info("No deadlocks detected."); + return; + } + + // Get information about deadlocked threads + ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(deadlockedThreadIds, true, true); + + // Build a deadlock report + StringBuilder deadlockReport = new StringBuilder("Deadlocks detected:\n"); + for (ThreadInfo threadInfo : threadInfos) { + deadlockReport.append("Thread Name: ").append(threadInfo.getThreadName()).append("\n") + .append("Thread State: ").append(threadInfo.getThreadState()).append("\n") + .append("Lock Info: ").append(threadInfo.getLockInfo()).append("\n") + .append("Blocked by: ").append(threadInfo.getLockName()).append("\n") + .append("Stack Trace: \n").append(formatStackTrace(threadInfo.getStackTrace())).append("\n"); + } + + // Log the deadlock report + LOG.warn(deadlockReport.toString()); + } + + /** + * Formats a stack trace for better readability. + * + * @param stackTraceElements the stack trace elements to format + * @return a formatted stack trace string + */ + private String formatStackTrace(StackTraceElement[] stackTraceElements) { + StringBuilder formattedStackTrace = new StringBuilder(); + for (StackTraceElement element : stackTraceElements) { + formattedStackTrace.append("\tat ").append(element).append("\n"); + } + return formattedStackTrace.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java new file mode 100644 index 00000000000000..4d6ffc46495ae3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.lock; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +/** + * A monitored version of ReentrantLock that provides additional monitoring capabilities + * for lock acquisition and release. + */ +public class MonitoredReentrantLock extends ReentrantLock { + private static final long serialVersionUID = 1L; + + // Monitor for tracking lock acquisition and release + private final AbstractMonitoredLock lockMonitor = new AbstractMonitoredLock() {}; + + /** + * Acquires the lock. + * Records the time when the lock is acquired. + */ + @Override + public void lock() { + super.lock(); + lockMonitor.afterLock(); + } + + /** + * Releases the lock. + * Records the time when the lock is released and logs the duration. + */ + @Override + public void unlock() { + lockMonitor.afterUnlock(); + super.unlock(); + } + + /** + * Tries to acquire the lock. + * Records the time when the lock attempt started and logs the result. + * + * @return true if the lock was acquired, false otherwise + */ + @Override + public boolean tryLock() { + long start = System.nanoTime(); // Record start time + boolean acquired = super.tryLock(); // Attempt to acquire the lock + lockMonitor.afterTryLock(acquired, start); // Log result and elapsed time + return acquired; + } + + /** + * Tries to acquire the lock within the specified time limit. + * Records the time when the lock attempt started and logs the result. + * + * @param timeout the time to wait for the lock + * @param unit the time unit of the timeout argument + * @return true if the lock was acquired, false if the waiting time elapsed before the lock was acquired + * @throws InterruptedException if the current thread is interrupted while waiting + */ + @Override + public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException { + long start = System.nanoTime(); // Record start time + boolean acquired = super.tryLock(timeout, unit); // Attempt to acquire the lock + lockMonitor.afterTryLock(acquired, start); // Log result and elapsed time + return acquired; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java new file mode 100644 index 00000000000000..4509340118abf1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.lock; + +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * A monitored version of ReentrantReadWriteLock that provides additional + * monitoring capabilities for read and write locks. + */ +public class MonitoredReentrantReadWriteLock extends ReentrantReadWriteLock { + private final AbstractMonitoredLock readLockMonitor = new AbstractMonitoredLock() { + }; + private final AbstractMonitoredLock writeLockMonitor = new AbstractMonitoredLock() { + }; + + /** + * Monitored read lock class that extends ReentrantReadWriteLock.ReadLock. + */ + public class ReadLock extends ReentrantReadWriteLock.ReadLock { + private static final long serialVersionUID = 1L; + + /** + * Constructs a new ReadLock instance. + * + * @param lock The ReentrantReadWriteLock this lock is associated with + */ + protected ReadLock(ReentrantReadWriteLock lock) { + super(lock); + } + + /** + * Acquires the read lock. + * Records the time when the lock is acquired. + */ + @Override + public void lock() { + super.lock(); + readLockMonitor.afterLock(); + } + + /** + * Releases the read lock. + * Records the time when the lock is released and logs the duration. + */ + @Override + public void unlock() { + readLockMonitor.afterUnlock(); + super.unlock(); + } + } + + /** + * Monitored write lock class that extends ReentrantReadWriteLock.WriteLock. + */ + public class WriteLock extends ReentrantReadWriteLock.WriteLock { + private static final long serialVersionUID = 1L; + + /** + * Constructs a new WriteLock instance. + * + * @param lock The ReentrantReadWriteLock this lock is associated with + */ + protected WriteLock(ReentrantReadWriteLock lock) { + super(lock); + } + + /** + * Acquires the write lock. + * Records the time when the lock is acquired. + */ + @Override + public void lock() { + super.lock(); + writeLockMonitor.afterLock(); + } + + /** + * Releases the write lock. + * Records the time when the lock is released and logs the duration. + */ + @Override + public void unlock() { + writeLockMonitor.afterUnlock(); + super.unlock(); + } + } + + /** + * Returns the read lock associated with this lock. + * + * @return The monitored read lock + */ + @Override + public ReadLock readLock() { + return new ReadLock(this); + } + + /** + * Returns the write lock associated with this lock. + * + * @return The monitored write lock + */ + @Override + public WriteLock writeLock() { + return new WriteLock(this); + } +} From 4efb2a28a26822f55e28075eec83d3a484110fa0 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Wed, 7 Aug 2024 19:55:51 +0800 Subject: [PATCH 2/7] [feat](lock)add deadlock detection tool and monitored lock implementations. --- .../java/org/apache/doris/common/Config.java | 13 ++++++ .../main/java/org/apache/doris/DorisFE.java | 11 ++++- .../org/apache/doris/alter/AlterHandler.java | 4 +- .../apache/doris/backup/BackupHandler.java | 11 +++-- .../apache/doris/backup/RepositoryMgr.java | 4 +- .../doris/binlog/BinlogConfigCache.java | 6 +-- .../apache/doris/binlog/BinlogManager.java | 6 +-- .../org/apache/doris/binlog/DBBinlog.java | 6 +-- .../org/apache/doris/binlog/TableBinlog.java | 6 +-- .../doris/blockrule/SqlBlockRuleMgr.java | 4 +- .../org/apache/doris/catalog/BrokerMgr.java | 4 +- .../doris/catalog/ColocateTableIndex.java | 4 +- .../doris/catalog/ColumnIdFlushDaemon.java | 4 +- .../java/org/apache/doris/catalog/MTMV.java | 8 ++-- .../org/apache/doris/catalog/Resource.java | 4 +- .../apache/doris/catalog/StorageVault.java | 4 +- .../apache/doris/catalog/StorageVaultMgr.java | 5 +-- .../java/org/apache/doris/catalog/Table.java | 8 ++-- .../java/org/apache/doris/catalog/Tablet.java | 4 +- .../doris/cloud/catalog/CloudPartition.java | 4 +- .../doris/cloud/rpc/MetaServiceProxy.java | 4 +- .../cloud/system/CloudSystemInfoService.java | 5 +-- .../common/lock/AbstractMonitoredLock.java | 8 ++-- .../doris/common/lock/DeadlockMonitor.java | 40 ++++++++----------- .../common/lock/MonitoredReentrantLock.java | 14 ++++++- .../lock/MonitoredReentrantReadWriteLock.java | 8 ++++ .../doris/common/util/ProfileManager.java | 12 +++--- .../doris/common/util/RuntimeProfile.java | 14 +++---- .../doris/consistency/ConsistencyChecker.java | 6 +-- .../apache/doris/datasource/CatalogMgr.java | 4 +- .../doris/datasource/ExternalDatabase.java | 6 +-- .../datasource/TablePartitionValues.java | 7 ++-- .../InsertOverwriteManager.java | 10 ++--- .../apache/doris/job/base/AbstractJob.java | 7 ++-- .../doris/job/extensions/mtmv/MTMVJob.java | 8 ++-- .../apache/doris/job/manager/JobManager.java | 4 +- .../doris/journal/bdbje/BDBEnvironment.java | 6 +-- .../org/apache/doris/load/DeleteHandler.java | 6 +-- .../java/org/apache/doris/load/ExportMgr.java | 4 +- .../main/java/org/apache/doris/load/Load.java | 6 +-- .../doris/load/StreamLoadRecordMgr.java | 4 +- .../org/apache/doris/load/loadv2/LoadJob.java | 4 +- .../apache/doris/load/loadv2/LoadManager.java | 4 +- .../load/routineload/RoutineLoadJob.java | 4 +- .../load/routineload/RoutineLoadManager.java | 4 +- .../doris/load/sync/SyncJobManager.java | 6 +-- .../sync/canal/CanalSyncDataConsumer.java | 7 ++-- .../sync/canal/CanalSyncDataReceiver.java | 6 +-- .../load/sync/canal/SyncCanalClient.java | 6 +-- .../mysql/authenticate/ldap/LdapManager.java | 4 +- .../apache/doris/mysql/privilege/Auth.java | 4 +- .../doris/mysql/privilege/PasswordPolicy.java | 4 +- .../nereids/jobs/load/LabelProcessor.java | 4 +- .../java/org/apache/doris/policy/Policy.java | 4 +- .../org/apache/doris/policy/PolicyMgr.java | 4 +- .../java/org/apache/doris/qe/Coordinator.java | 5 +-- .../org/apache/doris/qe/MultiLoadMgr.java | 5 +-- .../java/org/apache/doris/qe/VariableMgr.java | 5 +-- .../doris/qe/cache/CacheCoordinator.java | 5 +-- .../org/apache/doris/qe/help/HelpModule.java | 4 +- .../org/apache/doris/resource/TagManager.java | 4 +- .../resource/workloadgroup/QueryQueue.java | 4 +- .../workloadgroup/WorkloadGroupMgr.java | 4 +- .../WorkloadRuntimeStatusMgr.java | 4 +- .../WorkloadSchedPolicyMgr.java | 4 +- .../apache/doris/rpc/BackendServiceProxy.java | 4 +- .../doris/service/FrontendServiceImpl.java | 4 +- .../arrowflight/auth2/FlightAuthResult.java | 2 +- .../org/apache/doris/task/SerialExecutor.java | 5 ++- .../transaction/DatabaseTransactionMgr.java | 4 +- .../load/sync/canal/CanalSyncDataTest.java | 4 +- regression-test/pipeline/p0/conf/fe.conf | 3 ++ 72 files changed, 234 insertions(+), 204 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 3c8d0e0592afa9..a39e0e5c89cef3 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2987,4 +2987,17 @@ public static int metaServiceRpcRetryTimes() { //========================================================================== // end of cloud config //========================================================================== + //========================================================================== + // start of lock config + @ConfField(description = {"是否开启死锁检测", + "Whether to enable deadlock detection"}) + public static boolean enable_deadlock_detection = false; + + @ConfField(description = {"死锁检测间隔时间，单位分钟", + "Deadlock detection interval time, unit minute"}) + public static long deadlock_detection_interval_minute = 5; + + @ConfField(description = {"表示最大锁持有时间，超过该时间会打印告警日志，单位秒", + "Maximum lock hold time; logs a warning if exceeded"}) + public static long max_lock_hold_threshold_seconds = 30; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index de7feda719f7b3..d028f3aeae1437 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -27,6 +27,7 @@ import org.apache.doris.common.LogUtils; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.Version; +import org.apache.doris.common.lock.DeadlockMonitor; import org.apache.doris.common.util.JdkUtils; import org.apache.doris.common.util.NetUtils; import org.apache.doris.httpv2.HttpServer; @@ -60,6 +61,7 @@ import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; import java.nio.file.StandardOpenOption; +import java.util.concurrent.TimeUnit; public class DorisFE { private static final Logger LOG = LogManager.getLogger(DorisFE.class); @@ -95,6 +97,13 @@ public static void main(String[] args) { start(DORIS_HOME_DIR, PID_DIR, args, options); } + private static void startMonitor() { + if (Config.enable_deadlock_detection) { + DeadlockMonitor deadlockMonitor = new DeadlockMonitor(); + deadlockMonitor.startMonitoring(Config.deadlock_detection_interval_minute, TimeUnit.MINUTES); + } + } + // entrance for doris frontend public static void start(String dorisHomeDir, String pidDir, String[] args, StartupOptions options) { if (System.getenv("DORIS_LOG_TO_STDERR") != null) { @@ -214,7 +223,7 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star } ThreadPoolManager.registerAllThreadPoolMetric(); - + startMonitor(); while (true) { Thread.sleep(2000); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java index 4c4e1118670cce..41c2eaf2e815d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -33,6 +33,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.persist.RemoveAlterJobV2OperationLog; @@ -49,7 +50,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; public abstract class AlterHandler extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(AlterHandler.class); @@ -64,7 +64,7 @@ public abstract class AlterHandler extends MasterDaemon { * and this requires atomic operations. So the lock must be held to do this operations. * Operations like Get or Put do not need lock. */ - protected ReentrantLock lock = new ReentrantLock(); + protected MonitoredReentrantLock lock = new MonitoredReentrantLock(); public AlterHandler(String name) { this(name, FeConstants.default_scheduler_interval_millisecond); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 0cca0009d7e3c9..ab169929a833ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -46,6 +46,8 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.Pair; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantLock; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.fs.FileSystemFactory; @@ -80,9 +82,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -95,7 +94,7 @@ public class BackupHandler extends MasterDaemon implements Writable { private RepositoryMgr repoMgr = new RepositoryMgr(); // this lock is used for updating dbIdToBackupOrRestoreJobs - private final ReentrantLock jobLock = new ReentrantLock(); + private final MonitoredReentrantLock jobLock = new MonitoredReentrantLock(); // db id -> last 10(max_backup_restore_job_num_per_db) backup/restore jobs // Newly submitted job will replace the current job, only if current job is finished or cancelled. @@ -104,7 +103,7 @@ public class BackupHandler extends MasterDaemon implements Writable { private final Map> dbIdToBackupOrRestoreJobs = new HashMap<>(); // this lock is used for handling one backup or restore request at a time. - private ReentrantLock seqlock = new ReentrantLock(); + private MonitoredReentrantLock seqlock = new MonitoredReentrantLock(); private boolean isInit = false; @@ -114,7 +113,7 @@ public class BackupHandler extends MasterDaemon implements Writable { // this map not present in persist && only in fe master memory // one table only keep one snapshot info, only keep last private final Map localSnapshots = new HashMap<>(); - private ReadWriteLock localSnapshotsLock = new ReentrantReadWriteLock(); + private MonitoredReentrantReadWriteLock localSnapshotsLock = new MonitoredReentrantReadWriteLock(); public BackupHandler() { // for persist diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java index 853c1841449046..e5ccaf1a27865b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java @@ -22,6 +22,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.Daemon; import org.apache.doris.fs.remote.AzureFileSystem; import org.apache.doris.fs.remote.S3FileSystem; @@ -39,7 +40,6 @@ import java.io.IOException; import java.util.List; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; /* * A manager to manage all backup repositories @@ -52,7 +52,7 @@ public class RepositoryMgr extends Daemon implements Writable, GsonPostProcessab private ConcurrentMap repoNameMap = Maps.newConcurrentMap(); private ConcurrentMap repoIdMap = Maps.newConcurrentMap(); - private ReentrantLock lock = new ReentrantLock(); + private MonitoredReentrantLock lock = new MonitoredReentrantLock(); public RepositoryMgr() { super(Repository.class.getSimpleName(), 600 * 1000 /* 10min */); diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogConfigCache.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogConfigCache.java index b07f5e5d87ce41..0acda251c5111e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogConfigCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogConfigCache.java @@ -22,23 +22,23 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class BinlogConfigCache { private static final Logger LOG = LogManager.getLogger(BinlogConfigCache.class); private Map dbTableBinlogEnableMap; // db or table all use id - private ReentrantReadWriteLock lock; + private MonitoredReentrantReadWriteLock lock; public BinlogConfigCache() { dbTableBinlogEnableMap = new HashMap(); - lock = new ReentrantReadWriteLock(); + lock = new MonitoredReentrantReadWriteLock(); } // Get the binlog config of the specified db, return null if no such database diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java index 6fc2e3f813fb07..8d7c8cc2ad9ac9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.persist.AlterDatabasePropertyInfo; @@ -54,7 +55,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class BinlogManager { private static final int BUFFER_SIZE = 16 * 1024; @@ -67,12 +67,12 @@ public class BinlogManager { private static final Logger LOG = LogManager.getLogger(BinlogManager.class); - private ReentrantReadWriteLock lock; + private MonitoredReentrantReadWriteLock lock; private Map dbBinlogMap; private BinlogConfigCache binlogConfigCache; public BinlogManager() { - lock = new ReentrantReadWriteLock(); + lock = new MonitoredReentrantReadWriteLock(); dbBinlogMap = Maps.newHashMap(); binlogConfigCache = new BinlogConfigCache(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java index 502491004e5dab..eb0243dc7ee4cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.common.Pair; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.persist.DropPartitionInfo; import org.apache.doris.thrift.TBinlog; @@ -40,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.TreeSet; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class DBBinlog { @@ -50,7 +50,7 @@ public class DBBinlog { // The size of all binlogs. private long binlogSize; // guard for allBinlogs && tableBinlogMap - private ReentrantReadWriteLock lock; + private MonitoredReentrantReadWriteLock lock; // all binlogs contain table binlogs && create table binlog etc ... private TreeSet allBinlogs; // table binlogs @@ -70,7 +70,7 @@ public class DBBinlog { private BinlogConfigCache binlogConfigCache; public DBBinlog(BinlogConfigCache binlogConfigCache, TBinlog binlog) { - lock = new ReentrantReadWriteLock(); + lock = new MonitoredReentrantReadWriteLock(); this.dbId = binlog.getDbId(); this.binlogConfigCache = binlogConfigCache; this.binlogSize = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java index f3279b328c935f..07645b866be5fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.Pair; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.thrift.TBinlog; import org.apache.doris.thrift.TBinlogType; @@ -37,7 +38,6 @@ import java.util.Iterator; import java.util.List; import java.util.TreeSet; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class TableBinlog { private static final Logger LOG = LogManager.getLogger(TableBinlog.class); @@ -45,7 +45,7 @@ public class TableBinlog { private long dbId; private long tableId; private long binlogSize; - private ReentrantReadWriteLock lock; + private MonitoredReentrantReadWriteLock lock; private TreeSet binlogs; // Pair(commitSeq, timestamp), used for gc @@ -58,7 +58,7 @@ public TableBinlog(BinlogConfigCache binlogConfigCache, TBinlog binlog, long dbI this.dbId = dbId; this.tableId = tableId; this.binlogSize = 0; - lock = new ReentrantReadWriteLock(); + lock = new MonitoredReentrantReadWriteLock(); binlogs = Sets.newTreeSet(Comparator.comparingLong(TBinlog::getCommitSeq)); timestamps = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java index 2eba1d4fae3385..eb6514b16166e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java @@ -27,6 +27,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.SqlBlockUtil; import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.gson.GsonUtils; @@ -44,7 +45,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; /** @@ -53,7 +53,7 @@ public class SqlBlockRuleMgr implements Writable { private static final Logger LOG = LogManager.getLogger(SqlBlockRuleMgr.class); - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); @SerializedName(value = "nameToSqlBlockRuleMap") private Map nameToSqlBlockRuleMap = Maps.newConcurrentMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java index 5e4e7c7cbc84de..65bf667f833479 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java @@ -25,6 +25,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcNodeInterface; import org.apache.doris.common.proc.ProcResult; @@ -46,7 +47,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantLock; /** * Broker manager @@ -64,7 +64,7 @@ public class BrokerMgr { private final Map> brokersMap = Maps.newHashMap(); // { BrokerName -> { list of FsBroker } private final Map> brokerListMap = Maps.newHashMap(); - private final ReentrantLock lock = new ReentrantLock(); + private final MonitoredReentrantLock lock = new MonitoredReentrantLock(); private BrokerProcNode procNode = null; public BrokerMgr() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java index 28fc0ad55b8523..d253e4bae8c013 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java @@ -26,6 +26,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.persist.ColocatePersistInfo; @@ -57,7 +58,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; /** @@ -183,7 +183,7 @@ public static boolean isGlobalGroupName(String groupName) { // save some error msg of the group for show. no need to persist private Map group2ErrMsgs = Maps.newHashMap(); - private transient ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private transient MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); public ColocateTableIndex() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnIdFlushDaemon.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnIdFlushDaemon.java index 28007a8e666d47..8980158ce525ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnIdFlushDaemon.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnIdFlushDaemon.java @@ -19,6 +19,7 @@ import org.apache.doris.alter.AlterLightSchChangeHelper; import org.apache.doris.common.Config; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.persist.AlterLightSchemaChangeInfo; @@ -30,7 +31,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiConsumer; /** @@ -52,7 +52,7 @@ public class ColumnIdFlushDaemon extends MasterDaemon { public ColumnIdFlushDaemon() { super("colum-id-flusher", TimeUnit.HOURS.toMillis(1)); resultCollector = Maps.newHashMap(); - rwLock = new ReentrantReadWriteLock(); + rwLock = new MonitoredReentrantReadWriteLock(); if (Config.enable_convert_light_weight_schema_change) { flushFunc = this::doFlush; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 5097f83e2491e7..723fbe58836fb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -24,6 +24,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.job.common.TaskStatus; import org.apache.doris.job.extensions.mtmv.MTMVTask; @@ -62,12 +63,11 @@ import java.util.Map.Entry; import java.util.Optional; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class MTMV extends OlapTable { private static final Logger LOG = LogManager.getLogger(MTMV.class); - private ReentrantReadWriteLock mvRwLock; + private MonitoredReentrantReadWriteLock mvRwLock; @SerializedName("ri") private MTMVRefreshInfo refreshInfo; @@ -93,7 +93,7 @@ public class MTMV extends OlapTable { // For deserialization public MTMV() { type = TableType.MATERIALIZED_VIEW; - mvRwLock = new ReentrantReadWriteLock(true); + mvRwLock = new MonitoredReentrantReadWriteLock(true); } MTMV(MTMVParams params) { @@ -115,7 +115,7 @@ public MTMV() { this.mvPartitionInfo = params.mvPartitionInfo; this.relation = params.relation; this.refreshSnapshot = new MTMVRefreshSnapshot(); - mvRwLock = new ReentrantReadWriteLock(true); + mvRwLock = new MonitoredReentrantReadWriteLock(true); } public MTMVRefreshInfo getRefreshInfo() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index 0f0fd7d5de6d4f..ee57b784b59f3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -24,6 +24,7 @@ import org.apache.doris.common.io.DeepCopy; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.persist.gson.GsonPostProcessable; @@ -40,7 +41,6 @@ import java.io.IOException; import java.util.Map; import java.util.Map.Entry; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public abstract class Resource implements Writable, GsonPostProcessable { @@ -94,7 +94,7 @@ public enum ReferenceType { @SerializedName(value = "version") protected long version = -1; - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); public void writeLock() { lock.writeLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java index fb68581a00e850..d55f7ae0c4c18a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java @@ -23,6 +23,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -33,7 +34,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.IntStream; public abstract class StorageVault { @@ -65,7 +65,7 @@ public static StorageVaultType fromString(String storageVaultTypeType) { private boolean ifNotExists; private boolean setAsDefault; - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); public void writeLock() { lock.writeLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java index 2bec2839c86f32..f409f51141c0ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java @@ -26,6 +26,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.proto.InternalService.PAlterVaultSyncRequest; import org.apache.doris.rpc.BackendServiceProxy; @@ -43,8 +44,6 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class StorageVaultMgr { private static final Logger LOG = LogManager.getLogger(StorageVaultMgr.class); @@ -54,7 +53,7 @@ public class StorageVaultMgr { private Map vaultNameToVaultId = new HashMap<>(); - private ReadWriteLock rwLock = new ReentrantReadWriteLock(); + private MonitoredReentrantReadWriteLock rwLock = new MonitoredReentrantReadWriteLock(); private static final ExecutorService ALTER_BE_SYNC_THREAD_POOL = Executors.newFixedThreadPool(1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 77b6632fa6af21..59379f00baad0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -27,6 +27,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.QueryableReentrantReadWriteLock; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.common.util.Util; @@ -59,7 +60,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** @@ -88,7 +88,7 @@ public abstract class Table extends MetaObject implements Writable, TableIf, Gso // Used for queuing commit transactifon tasks to avoid fdb transaction conflicts, // especially to reduce conflicts when obtaining delete bitmap update locks for // MoW table - protected ReentrantLock commitLock; + protected MonitoredReentrantLock commitLock; /* * fullSchema and nameToColumn should contains all columns, both visible and shadow. @@ -138,7 +138,7 @@ public Table(TableType type) { if (Config.check_table_lock_leaky) { this.readLockThreads = Maps.newConcurrentMap(); } - this.commitLock = new ReentrantLock(true); + this.commitLock = new MonitoredReentrantLock(true); } public Table(long id, String tableName, TableType type, List fullSchema) { @@ -163,7 +163,7 @@ public Table(long id, String tableName, TableType type, List fullSchema) if (Config.check_table_lock_leaky) { this.readLockThreads = Maps.newConcurrentMap(); } - this.commitLock = new ReentrantLock(true); + this.commitLock = new MonitoredReentrantLock(true); } public void markDropped() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java index c32b2bcbfa9e3e..e516b8e2306331 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java @@ -27,6 +27,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.resource.Tag; import org.apache.doris.system.Backend; @@ -51,7 +52,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -98,7 +98,7 @@ public enum TabletStatus { private long cooldownReplicaId = -1; @SerializedName(value = "ctm", alternate = {"cooldownTerm"}) private long cooldownTerm = -1; - private ReentrantReadWriteLock cooldownConfLock = new ReentrantReadWriteLock(); + private MonitoredReentrantReadWriteLock cooldownConfLock = new MonitoredReentrantReadWriteLock(); // last time that the tablet checker checks this tablet. // no need to persist diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java index 5036a0e01c4464..3b1c987c0c54bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java @@ -25,6 +25,7 @@ import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; import org.apache.doris.cloud.rpc.VersionHelper; import org.apache.doris.common.Config; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.qe.ConnectContext; @@ -39,7 +40,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** @@ -54,7 +54,7 @@ public class CloudPartition extends Partition { @SerializedName(value = "tableId") private long tableId; - private ReentrantLock lock = new ReentrantLock(true); + private MonitoredReentrantLock lock = new MonitoredReentrantLock(true); public CloudPartition(long id, String name, MaterializedIndex baseIndex, DistributionInfo distributionInfo, long dbId, long tableId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index 5f17692180b353..66b0ddf611a925 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -19,6 +19,7 @@ import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.common.Config; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.rpc.RpcException; import com.google.common.collect.Maps; @@ -28,7 +29,6 @@ import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; public class MetaServiceProxy { private static final Logger LOG = LogManager.getLogger(MetaServiceProxy.class); @@ -36,7 +36,7 @@ public class MetaServiceProxy { // use exclusive lock to make sure only one thread can add or remove client from // serviceMap. // use concurrent map to allow access serviceMap in multi thread. - private ReentrantLock lock = new ReentrantLock(); + private MonitoredReentrantLock lock = new MonitoredReentrantLock(); private final Map serviceMap; static { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java index 48728efb003aba..11e0fa70de828a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java @@ -30,6 +30,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.ha.FrontendNodeType; import org.apache.doris.metric.MetricRepo; import org.apache.doris.qe.ConnectContext; @@ -57,15 +58,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class CloudSystemInfoService extends SystemInfoService { private static final Logger LOG = LogManager.getLogger(CloudSystemInfoService.class); // use rw lock to make sure only one thread can change clusterIdToBackend and clusterNameToId - private static final ReadWriteLock rwlock = new ReentrantReadWriteLock(); + private static final MonitoredReentrantReadWriteLock rwlock = new MonitoredReentrantReadWriteLock(); private static final Lock rlock = rwlock.readLock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java index da883b69b1c6bd..964aeafa17743a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java @@ -17,6 +17,8 @@ package org.apache.doris.common.lock; +import org.apache.doris.common.Config; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,7 +30,8 @@ * a specified timeout. */ public abstract class AbstractMonitoredLock { - protected static final long HOLD_TIMEOUT = 5000; // Lock hold timeout in milliseconds + // Lock hold timeout in milliseconds + protected static final long HOLD_TIMEOUT = Config.max_lock_hold_threshold_seconds * 1000L; private static final Logger LOG = LoggerFactory.getLogger(AbstractMonitoredLock.class); // Thread-local variable to store the lock start time @@ -52,9 +55,8 @@ protected void afterUnlock() { if (startTime != null) { long lockHoldTimeNanos = System.nanoTime() - startTime; long lockHoldTimeMs = lockHoldTimeNanos >> 20; - Thread currentThread = Thread.currentThread(); - if (lockHoldTimeMs > HOLD_TIMEOUT) { + Thread currentThread = Thread.currentThread(); String stackTrace = getThreadStackTrace(currentThread.getStackTrace()); LOG.warn("Thread ID: {}, Thread Name: {} - Lock held for {} ms, exceeding hold timeout of {} ms \n" + "Thread stack trace:\n{}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java index 0c22ad17607f48..cc6454a910e942 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java @@ -17,6 +17,7 @@ package org.apache.doris.common.lock; +import com.google.gson.Gson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,13 +38,14 @@ */ public class DeadlockMonitor { private static final Logger LOG = LoggerFactory.getLogger(DeadlockMonitor.class); - + private final Gson gson; private final ThreadMXBean threadMXBean; private final ScheduledExecutorService scheduler; public DeadlockMonitor() { this.threadMXBean = ManagementFactory.getThreadMXBean(); this.scheduler = Executors.newScheduledThreadPool(1); + this.gson = new Gson(); } /** @@ -53,7 +55,7 @@ public DeadlockMonitor() { * @param unit the time unit of the period parameter */ public void startMonitoring(long period, TimeUnit unit) { - scheduler.scheduleAtFixedRate(this::detectAndReportDeadlocks, 0, period, unit); + scheduler.scheduleAtFixedRate(this::detectAndReportDeadlocks, 5, period, unit); } /** @@ -65,7 +67,9 @@ public void detectAndReportDeadlocks() { // Check if there are no deadlocked threads if (deadlockedThreadIds == null || deadlockedThreadIds.length == 0) { - LOG.info("No deadlocks detected."); + if (LOG.isDebugEnabled()) { + LOG.debug("No deadlocks detected."); + } return; } @@ -73,30 +77,20 @@ public void detectAndReportDeadlocks() { ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(deadlockedThreadIds, true, true); // Build a deadlock report - StringBuilder deadlockReport = new StringBuilder("Deadlocks detected:\n"); - for (ThreadInfo threadInfo : threadInfos) { - deadlockReport.append("Thread Name: ").append(threadInfo.getThreadName()).append("\n") - .append("Thread State: ").append(threadInfo.getThreadState()).append("\n") - .append("Lock Info: ").append(threadInfo.getLockInfo()).append("\n") - .append("Blocked by: ").append(threadInfo.getLockName()).append("\n") - .append("Stack Trace: \n").append(formatStackTrace(threadInfo.getStackTrace())).append("\n"); - } // Log the deadlock report - LOG.warn(deadlockReport.toString()); + DeadlockThreadsInfo deadlockThreadsInfo = new DeadlockThreadsInfo(threadInfos); + LOG.warn("Deadlocks detected {}", gson.toJson(deadlockThreadsInfo)); } - /** - * Formats a stack trace for better readability. - * - * @param stackTraceElements the stack trace elements to format - * @return a formatted stack trace string - */ - private String formatStackTrace(StackTraceElement[] stackTraceElements) { - StringBuilder formattedStackTrace = new StringBuilder(); - for (StackTraceElement element : stackTraceElements) { - formattedStackTrace.append("\tat ").append(element).append("\n"); + private static class DeadlockThreadsInfo { + private String[] threadInfos; + + private DeadlockThreadsInfo(ThreadInfo[] threadInfos) { + this.threadInfos = new String[threadInfos.length]; + for (int i = 0; i < threadInfos.length; i++) { + this.threadInfos[i] = threadInfos[i].toString(); + } } - return formattedStackTrace.toString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java index 4d6ffc46495ae3..263f37bca53f5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java @@ -28,7 +28,17 @@ public class MonitoredReentrantLock extends ReentrantLock { private static final long serialVersionUID = 1L; // Monitor for tracking lock acquisition and release - private final AbstractMonitoredLock lockMonitor = new AbstractMonitoredLock() {}; + private final AbstractMonitoredLock lockMonitor = new AbstractMonitoredLock() { + }; + + // Constructor for creating a monitored lock with fairness option + public MonitoredReentrantLock(boolean fair) { + super(fair); + } + + // Constructor for creating a monitored lock with fairness option + public MonitoredReentrantLock() { + } /** * Acquires the lock. @@ -69,7 +79,7 @@ public boolean tryLock() { * Records the time when the lock attempt started and logs the result. * * @param timeout the time to wait for the lock - * @param unit the time unit of the timeout argument + * @param unit the time unit of the timeout argument * @return true if the lock was acquired, false if the waiting time elapsed before the lock was acquired * @throws InterruptedException if the current thread is interrupted while waiting */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java index 4509340118abf1..68f48eca93a8d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java @@ -29,6 +29,14 @@ public class MonitoredReentrantReadWriteLock extends ReentrantReadWriteLock { private final AbstractMonitoredLock writeLockMonitor = new AbstractMonitoredLock() { }; + // Constructor for creating a monitored lock with fairness option + public MonitoredReentrantReadWriteLock(boolean fair) { + super(fair); + } + + public MonitoredReentrantReadWriteLock() { + } + /** * Monitored read lock class that extends ReentrantReadWriteLock.ReadLock. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java index 306b5f1b4e621a..ef8ac5235702b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.profile.ExecutionProfile; import org.apache.doris.common.profile.MultiProfileTreeBuilder; import org.apache.doris.common.profile.Profile; @@ -60,9 +61,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; /* * if you want to visit the attribute(such as queryID,defaultDb) @@ -131,9 +129,9 @@ public void deleteFromStorage() { boolean isProfileLoaded = false; // only protect queryIdDeque; queryIdToProfileMap is concurrent, no need to protect - private ReentrantReadWriteLock lock; - private ReadLock readLock; - private WriteLock writeLock; + private MonitoredReentrantReadWriteLock lock; + private MonitoredReentrantReadWriteLock.ReadLock readLock; + private MonitoredReentrantReadWriteLock.WriteLock writeLock; // profile id is long string for brocker load // is TUniqueId for others. @@ -159,7 +157,7 @@ public static ProfileManager getInstance() { // The visiablity of ProfileManager() is package level, so that we can write ut for it. ProfileManager() { - lock = new ReentrantReadWriteLock(true); + lock = new MonitoredReentrantReadWriteLock(true); readLock = lock.readLock(); writeLock = lock.writeLock(); queryIdToProfileMap = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java index 60207b49172ba0..d2da55c2679523 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.Reference; import org.apache.doris.common.io.Text; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TCounter; @@ -45,7 +46,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * It is accessed by two kinds of thread, one is to create this RuntimeProfile @@ -67,19 +67,19 @@ public class RuntimeProfile { private Map infoStrings = Maps.newHashMap(); @SerializedName(value = "infoStringsDisplayOrder") private List infoStringsDisplayOrder = Lists.newArrayList(); - private transient ReentrantReadWriteLock infoStringsLock = new ReentrantReadWriteLock(); + private transient MonitoredReentrantReadWriteLock infoStringsLock = new MonitoredReentrantReadWriteLock(); @SerializedName(value = "counterMap") private Map counterMap = Maps.newConcurrentMap(); @SerializedName(value = "childCounterMap") private Map> childCounterMap = Maps.newConcurrentMap(); // protect TreeSet in ChildCounterMap - private transient ReentrantReadWriteLock counterLock = new ReentrantReadWriteLock(); + private transient MonitoredReentrantReadWriteLock counterLock = new MonitoredReentrantReadWriteLock(); @SerializedName(value = "childMap") private Map childMap = Maps.newConcurrentMap(); @SerializedName(value = "childList") private LinkedList> childList = Lists.newLinkedList(); - private transient ReentrantReadWriteLock childLock = new ReentrantReadWriteLock(); + private transient MonitoredReentrantReadWriteLock childLock = new MonitoredReentrantReadWriteLock(); @SerializedName(value = "planNodeInfos") private List planNodeInfos = Lists.newArrayList(); @@ -126,9 +126,9 @@ public RuntimeProfile(String name, int nodeId) { } private void init() { - this.infoStringsLock = new ReentrantReadWriteLock(); - this.childLock = new ReentrantReadWriteLock(); - this.counterLock = new ReentrantReadWriteLock(); + this.infoStringsLock = new MonitoredReentrantReadWriteLock(); + this.childLock = new MonitoredReentrantReadWriteLock(); + this.counterLock = new MonitoredReentrantReadWriteLock(); } public static RuntimeProfile read(DataInput input) throws IOException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java b/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java index 471e235684e390..73394c92946082 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.common.Config; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.consistency.CheckConsistencyJob.JobState; @@ -47,7 +48,6 @@ import java.util.Map; import java.util.PriorityQueue; import java.util.Queue; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class ConsistencyChecker extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(ConsistencyChecker.class); @@ -69,7 +69,7 @@ public class ConsistencyChecker extends MasterDaemon { * * if reversal is inevitable. use db.tryLock() instead to avoid dead lock */ - private ReentrantReadWriteLock jobsLock; + private MonitoredReentrantReadWriteLock jobsLock; private int startTime; private int endTime; @@ -78,7 +78,7 @@ public ConsistencyChecker() { super("consistency checker"); jobs = Maps.newHashMap(); - jobsLock = new ReentrantReadWriteLock(); + jobsLock = new MonitoredReentrantReadWriteLock(); if (!initWorkTime()) { LOG.error("failed to init time in ConsistencyChecker. exit"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 8ed602d0eb850f..b8c88c9c08e11d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -41,6 +41,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; @@ -70,7 +71,6 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import java.util.stream.Collectors; @@ -90,7 +90,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { private static final String YES = "yes"; - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); @SerializedName(value = "idToCatalog") private final Map>> idToCatalog = Maps.newConcurrentMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 2919633858f90c..b564a17ce8687b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -28,6 +28,7 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaTable; @@ -58,7 +59,6 @@ import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Base class of external database. @@ -69,7 +69,7 @@ public abstract class ExternalDatabase implements DatabaseIf, Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(ExternalDatabase.class); - protected ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); + protected MonitoredReentrantReadWriteLock rwLock = new MonitoredReentrantReadWriteLock(true); @SerializedName(value = "id") protected long id; @@ -446,7 +446,7 @@ public void gsonPostProcess() throws IOException { } } idToTbl = tmpIdToTbl; - rwLock = new ReentrantReadWriteLock(true); + rwLock = new MonitoredReentrantReadWriteLock(true); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/TablePartitionValues.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/TablePartitionValues.java index 60765d705d554d..d5e8a39e605a8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/TablePartitionValues.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/TablePartitionValues.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.planner.ColumnBound; import org.apache.doris.planner.ListPartitionPrunerV2; import org.apache.doris.planner.PartitionPrunerV2Base.UniqueId; @@ -43,15 +44,13 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @Data public class TablePartitionValues { public static final String HIVE_DEFAULT_PARTITION = "__HIVE_DEFAULT_PARTITION__"; - private final ReadWriteLock readWriteLock; + private final MonitoredReentrantReadWriteLock readWriteLock; private long lastUpdateTimestamp; private long nextPartitionId; private final Map idToPartitionItem; @@ -68,7 +67,7 @@ public class TablePartitionValues { private Map> singleUidToColumnRangeMap; public TablePartitionValues() { - readWriteLock = new ReentrantReadWriteLock(); + readWriteLock = new MonitoredReentrantReadWriteLock(); lastUpdateTimestamp = 0; nextPartitionId = 0; idToPartitionItem = new HashMap<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java index 81524ae020810e..ccb65d3201c97e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java @@ -22,6 +22,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.insertoverwrite.InsertOverwriteLog.InsertOverwriteOpType; import org.apache.doris.persist.gson.GsonUtils; @@ -40,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.concurrent.locks.ReentrantLock; public class InsertOverwriteManager extends MasterDaemon implements Writable { private static final Logger LOG = LogManager.getLogger(InsertOverwriteManager.class); @@ -57,7 +57,7 @@ public class InsertOverwriteManager extends MasterDaemon implements Writable { // for one task group, there may be different requests about changing a partition to new. // but we only change one time and save the relations in partitionPairs. they're protected by taskLocks @SerializedName(value = "taskLocks") - private Map taskLocks = Maps.newConcurrentMap(); + private Map taskLocks = Maps.newConcurrentMap(); // >. no need concern which task it belongs to. @SerializedName(value = "partitionPairs") private Map> partitionPairs = Maps.newConcurrentMap(); @@ -94,7 +94,7 @@ public long registerTask(long dbId, long tableId, List tempPartitionName public long registerTaskGroup() { long groupId = Env.getCurrentEnv().getNextId(); taskGroups.put(groupId, new ArrayList()); - taskLocks.put(groupId, new ReentrantLock()); + taskLocks.put(groupId, new MonitoredReentrantLock()); partitionPairs.put(groupId, Maps.newConcurrentMap()); return groupId; } @@ -141,7 +141,7 @@ public void recordPartitionPairs(long groupId, List oldIds, List new } // lock is a symbol of TaskGroup exist. if not, means already failed. - public ReentrantLock getLock(long groupId) { + public MonitoredReentrantLock getLock(long groupId) { return taskLocks.get(groupId); } @@ -149,7 +149,7 @@ public ReentrantLock getLock(long groupId) { // it will cause ConcurrentModification or NullPointer. public void taskGroupFail(long groupId) { LOG.info("insert overwrite auto detect partition task group [" + groupId + "] failed"); - ReentrantLock lock = getLock(groupId); + MonitoredReentrantLock lock = getLock(groupId); lock.lock(); try { // will rollback temp partitions in `taskFail` diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java index 3f595d6daf5362..98291cdc6fa742 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.common.util.TimeUtils; @@ -52,8 +53,6 @@ import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @Data @@ -146,7 +145,7 @@ public AbstractJob(Long jobId, String jobName, JobStatus jobStatus, private CopyOnWriteArrayList runningTasks = new CopyOnWriteArrayList<>(); - private Lock createTaskLock = new ReentrantLock(); + private MonitoredReentrantLock createTaskLock = new MonitoredReentrantLock(); @Override public void cancelAllTasks() throws JobException { @@ -288,7 +287,7 @@ public static AbstractJob readFields(DataInput in) throws IOException { String jsonJob = Text.readString(in); AbstractJob job = GsonUtils.GSON.fromJson(jsonJob, AbstractJob.class); job.runningTasks = new CopyOnWriteArrayList(); - job.createTaskLock = new ReentrantLock(); + job.createTaskLock = new MonitoredReentrantLock(); return job; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java index 5d7cf4435b9c7a..a4e1c8e9867a7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java @@ -28,6 +28,7 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.job.base.AbstractJob; @@ -51,11 +52,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class MTMVJob extends AbstractJob { private static final Logger LOG = LogManager.getLogger(MTMVJob.class); - private ReentrantReadWriteLock jobRwLock; + private MonitoredReentrantReadWriteLock jobRwLock; private static final ShowResultSetMetaData JOB_META_DATA = ShowResultSetMetaData.builder() @@ -108,14 +108,14 @@ public class MTMVJob extends AbstractJob { private long mtmvId; public MTMVJob() { - jobRwLock = new ReentrantReadWriteLock(true); + jobRwLock = new MonitoredReentrantReadWriteLock(true); } public MTMVJob(long dbId, long mtmvId) { this.dbId = dbId; this.mtmvId = mtmvId; super.setCreateTimeMs(System.currentTimeMillis()); - jobRwLock = new ReentrantReadWriteLock(true); + jobRwLock = new MonitoredReentrantReadWriteLock(true); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java index 39646bab18f785..7b9d849f3b9bf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java @@ -29,6 +29,7 @@ import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.datasource.InternalCatalog; @@ -55,7 +56,6 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @Log4j2 @@ -67,7 +67,7 @@ public class JobManager, C> implements Writable { // lock for job // lock is private and must use after db lock - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java index ff162b0c9fa522..55a1ff3983bdbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.ha.BDBHA; import org.apache.doris.ha.BDBStateChangeListener; import org.apache.doris.ha.FrontendNodeType; @@ -58,7 +59,6 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; import java.util.stream.Collectors; @@ -79,7 +79,7 @@ public class BDBEnvironment { private ReplicationConfig replicationConfig; private DatabaseConfig dbConfig; private Database epochDB = null; // used for fencing - private ReentrantReadWriteLock lock; + private MonitoredReentrantReadWriteLock lock; private List openedDatabases; private final boolean isElectable; @@ -87,7 +87,7 @@ public class BDBEnvironment { public BDBEnvironment(boolean isElectable, boolean metadataFailureRecovery) { openedDatabases = new ArrayList(); - this.lock = new ReentrantReadWriteLock(true); + this.lock = new MonitoredReentrantReadWriteLock(true); this.isElectable = isElectable; this.metadataFailureRecovery = metadataFailureRecovery; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 299dac295dc909..1d96eea960fbec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -27,6 +27,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; @@ -54,7 +55,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.UUID; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class DeleteHandler implements Writable { private static final Logger LOG = LogManager.getLogger(DeleteHandler.class); @@ -66,12 +66,12 @@ public class DeleteHandler implements Writable { @SerializedName(value = "dbToDeleteInfos") private final Map> dbToDeleteInfos; - private final ReentrantReadWriteLock lock; + private final MonitoredReentrantReadWriteLock lock; public DeleteHandler() { idToDeleteJob = Maps.newConcurrentMap(); dbToDeleteInfos = Maps.newConcurrentMap(); - lock = new ReentrantReadWriteLock(); + lock = new MonitoredReentrantReadWriteLock(); } public void readLock() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java index bce0c5c4d9f195..85839c8716a30d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java @@ -32,6 +32,7 @@ import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.OrderByPair; import org.apache.doris.common.util.TimeUtils; @@ -56,7 +57,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -68,7 +68,7 @@ public class ExportMgr { // lock for export job // lock is private and must use after db lock - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); public ExportMgr() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index 0323f3ea62ee15..cdbbf1628107be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -62,6 +62,7 @@ import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.ExprUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; @@ -93,7 +94,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class Load { private static final Logger LOG = LogManager.getLogger(Load.class); @@ -120,7 +120,7 @@ public class Load { // lock for load job // lock is private and must use after db lock - private ReentrantReadWriteLock lock; + private MonitoredReentrantReadWriteLock lock; static { Set pendingDestStates = Sets.newHashSet(); @@ -180,7 +180,7 @@ public Load() { idToQuorumFinishedLoadJob = Maps.newLinkedHashMap(); loadingPartitionIds = Sets.newHashSet(); dbToMiniLabels = Maps.newHashMap(); - lock = new ReentrantReadWriteLock(true); + lock = new MonitoredReentrantReadWriteLock(true); } public void readLock() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java index 1e1dc192ceeac1..0a60bcbad41422 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java @@ -26,6 +26,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; @@ -61,7 +62,6 @@ import java.util.PriorityQueue; import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class StreamLoadRecordMgr extends MasterDaemon { @@ -107,7 +107,7 @@ public int compare(StreamLoadItem s1, StreamLoadItem s2) { Queue streamLoadRecordHeap = new PriorityQueue<>(new StreamLoadComparator()); private Map> dbIdToLabelToStreamLoadRecord = Maps.newConcurrentMap(); - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); public StreamLoadRecordMgr(String name, long intervalMs) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index 3268fb3c464836..bad440b7c1f2b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -39,6 +39,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.common.util.TimeUtils; @@ -81,7 +82,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; public abstract class LoadJob extends AbstractTxnStateChangeCallback implements LoadTaskCallback, Writable, GsonPostProcessable { @@ -134,7 +134,7 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback // During committing, the load job could not be cancelled. protected boolean isCommitting = false; - protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + protected MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); // this request id is only used for checking if a load begin request is a duplicate request. protected TUniqueId requestId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java index 1026689d8f1448..e87ca2bf8b4902 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java @@ -40,6 +40,7 @@ import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.datasource.InternalCatalog; @@ -83,7 +84,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -101,7 +101,7 @@ public class LoadManager implements Writable { protected Map>> dbIdToLabelToLoadJobs = Maps.newConcurrentMap(); protected LoadJobScheduler loadJobScheduler; - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); private MysqlLoadManager mysqlLoadManager; private TokenManager tokenManager; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index a25cd99985892e..ef0cc2a3111dc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -42,6 +42,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; @@ -95,7 +96,6 @@ import java.util.Optional; import java.util.Queue; import java.util.UUID; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Routine load job is a function which stream load data from streaming medium to doris. @@ -276,7 +276,7 @@ public boolean isFinalState() { @SerializedName("cm") protected String comment = ""; - protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + protected MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete protected Expr deleteCondition; // TODO(ml): error sample diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index 8b0ebab3fbe7d5..234c1228140fb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -40,6 +40,7 @@ import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.datasource.InternalCatalog; @@ -73,7 +74,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -89,7 +89,7 @@ public class RoutineLoadManager implements Writable { private ConcurrentHashMap multiLoadTaskTxnIdToRoutineLoadJobId = new ConcurrentHashMap<>(); - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/SyncJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/SyncJobManager.java index 9455d2c9ea623a..1d4c27f3c7cdbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/SyncJobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/SyncJobManager.java @@ -27,6 +27,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.load.sync.canal.CanalDestination; @@ -49,7 +50,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -60,12 +60,12 @@ public class SyncJobManager implements Writable { private Map>> dbIdToJobNameToSyncJobs; - private ReentrantReadWriteLock lock; + private MonitoredReentrantReadWriteLock lock; public SyncJobManager() { idToSyncJob = Maps.newConcurrentMap(); dbIdToJobNameToSyncJobs = Collections.synchronizedMap(Maps.newLinkedHashMap()); - lock = new ReentrantReadWriteLock(true); + lock = new MonitoredReentrantReadWriteLock(true); } public void addDataSyncJob(CreateDataSyncJobStmt stmt) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java index 25500772248992..32af4afc33f3d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Status; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.load.sync.SyncChannelHandle; import org.apache.doris.load.sync.SyncDataConsumer; import org.apache.doris.load.sync.SyncFailMsg; @@ -53,7 +54,6 @@ import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; public class CanalSyncDataConsumer extends SyncDataConsumer { private static Logger logger = LogManager.getLogger(CanalSyncDataConsumer.class); @@ -70,7 +70,7 @@ public class CanalSyncDataConsumer extends SyncDataConsumer { private PositionMeta positionMeta; private LinkedBlockingQueue> dataBlockingQueue; private SyncChannelHandle handle; - private ReentrantLock getLock; + private MonitoredReentrantLock getLock; private int sleepTimeMs; private long commitIntervalSecond; @@ -82,7 +82,8 @@ public void setChannels(Map idToChannels) { this.idToChannels = idToChannels; } - public CanalSyncDataConsumer(CanalSyncJob syncJob, CanalConnector connector, ReentrantLock getLock, boolean debug) { + public CanalSyncDataConsumer(CanalSyncJob syncJob, CanalConnector connector, + MonitoredReentrantLock getLock, boolean debug) { super(debug); this.syncJob = syncJob; this.connector = connector; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataReceiver.java index c3ebcbc4928739..f894bae7a4aa62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataReceiver.java @@ -17,6 +17,7 @@ package org.apache.doris.load.sync.canal; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.load.sync.SyncDataReceiver; import com.alibaba.otter.canal.client.CanalConnector; @@ -25,21 +26,20 @@ import org.apache.logging.log4j.Logger; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; public class CanalSyncDataReceiver extends SyncDataReceiver { private static Logger LOG = LogManager.getLogger(CanalSyncDataReceiver.class); private CanalSyncJob syncJob; private CanalConnector connector; - private ReentrantLock getLock; + private MonitoredReentrantLock getLock; private CanalSyncDataConsumer consumer; private String destination; private String filter; private long sleepTimeMs; public CanalSyncDataReceiver(CanalSyncJob syncJob, CanalConnector connector, String destination, String filter, - CanalSyncDataConsumer consumer, int readBatchSize, ReentrantLock getLock) { + CanalSyncDataConsumer consumer, int readBatchSize, MonitoredReentrantLock getLock) { super(readBatchSize); this.syncJob = syncJob; this.connector = connector; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java index d41def0465a5c4..847f24e0b30cc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java @@ -17,6 +17,7 @@ package org.apache.doris.load.sync.canal; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.load.sync.SyncChannel; import com.alibaba.otter.canal.client.CanalConnector; @@ -27,7 +28,6 @@ import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantLock; public class SyncCanalClient { protected static Logger logger = LogManager.getLogger(SyncCanalClient.class); @@ -39,8 +39,8 @@ public class SyncCanalClient { // channel id -> channel private final Map idToChannels; - protected ReentrantLock lock = new ReentrantLock(true); - protected ReentrantLock getLock = new ReentrantLock(); + protected MonitoredReentrantLock lock = new MonitoredReentrantLock(true); + protected MonitoredReentrantLock getLock = new MonitoredReentrantLock(); protected void lock() { lock.lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapManager.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapManager.java index 5f6003cd6c146c..e8ebccc46e0008 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapManager.java @@ -25,6 +25,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.LdapConfig; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.mysql.authenticate.AuthenticateType; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.PrivBitSet; @@ -42,7 +43,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Encapsulates LDAP service interfaces and caches user LDAP information. @@ -56,7 +56,7 @@ public class LdapManager { private final Map ldapUserInfoCache = Maps.newHashMap(); - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java index 2073e33cb4fdfc..9e1b7197c28996 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java @@ -54,6 +54,7 @@ import org.apache.doris.common.PatternMatcherException; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.MysqlPassword; import org.apache.doris.mysql.authenticate.AuthenticateType; @@ -87,7 +88,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @@ -116,7 +116,7 @@ public class Auth implements Writable { private PasswordPolicyManager passwdPolicyManager = new PasswordPolicyManager(); - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PasswordPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PasswordPolicy.java index 367eec89827274..080d413c051157 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PasswordPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PasswordPolicy.java @@ -23,6 +23,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.persist.gson.GsonUtils; @@ -43,7 +44,6 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Password policy for a specific user. @@ -56,7 +56,7 @@ public class PasswordPolicy implements Writable { private static final Logger LOG = LogManager.getLogger(PasswordPolicy.class); - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); private static final String EXPIRATION_SECONDS = "password_policy.expiration_seconds"; private static final String PASSWORD_CREATION_TIME = "password_policy.password_creation_time"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/load/LabelProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/load/LabelProcessor.java index 88454eaecdc4e0..c0ea85dc9f2177 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/load/LabelProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/load/LabelProcessor.java @@ -23,6 +23,7 @@ import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.job.exception.JobException; import org.apache.doris.job.extensions.insert.InsertJob; @@ -33,7 +34,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; /** @@ -41,7 +41,7 @@ */ public class LabelProcessor { private final Map>> dbIdToLabelToLoadJobs = new ConcurrentHashMap<>(); - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java index 01c5399d4ab974..c99158cfcf3a35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java @@ -24,6 +24,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -38,7 +39,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Base class for Policy. @@ -60,7 +60,7 @@ public abstract class Policy implements Writable, GsonPostProcessable { @SerializedName(value = "version") protected long version = -1; - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); public void writeLock() { lock.writeLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java index 6e8bd4f08cb2f7..6836bd694e3b0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java @@ -35,6 +35,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ShowResultSet; @@ -63,7 +64,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; /** @@ -72,7 +72,7 @@ public class PolicyMgr implements Writable { private static final Logger LOG = LogManager.getLogger(PolicyMgr.class); - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); @SerializedName(value = "typeToPolicyMap") private Map> typeToPolicyMap = Maps.newConcurrentMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 4ee46d3bec3c38..16caea9f94bbc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -29,6 +29,7 @@ import org.apache.doris.common.Status; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.profile.ExecutionProfile; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.common.util.DebugUtil; @@ -161,8 +162,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -206,7 +205,7 @@ public class Coordinator implements CoordInterface { private TNetworkAddress currentConnectFE; // protects all fields below - private final Lock lock = new ReentrantLock(); + private final MonitoredReentrantLock lock = new MonitoredReentrantLock(); // If true, the query is done returning all results. It is possible that the // coordinator still needs to wait for cleanup on remote fragments (e.g. queries diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MultiLoadMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MultiLoadMgr.java index 63f6de760cb102..c182b7f10814b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MultiLoadMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MultiLoadMgr.java @@ -35,6 +35,7 @@ import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.load.EtlJobType; import org.apache.doris.load.loadv2.JobState; @@ -60,8 +61,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; // Class used to record state of multi-load operation @@ -69,7 +68,7 @@ public class MultiLoadMgr { private static final Logger LOG = LogManager.getLogger(MultiLoadMgr.class); private Map infoMap = Maps.newHashMap(); - private ReadWriteLock lock = new ReentrantReadWriteLock(true); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); // Start multi-load transaction. // Label is the only need parameter, maybe other properties? diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 1b1a317f5f0bb5..3461061b56f9d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -31,6 +31,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.VariableAnnotation; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.persist.GlobalVarPersistInfo; @@ -63,8 +64,6 @@ import java.util.Map; import java.util.StringJoiner; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.annotation.Nullable; /** @@ -138,7 +137,7 @@ public class VariableMgr { private static ImmutableMap ctxByVarNameForCkpt; // Global read/write lock to protect access of globalSessionVariable. - private static final ReadWriteLock rwlock = new ReentrantReadWriteLock(); + private static final MonitoredReentrantReadWriteLock rwlock = new MonitoredReentrantReadWriteLock(); private static final Lock rlock = rwlock.readLock(); private static final Lock wlock = rwlock.writeLock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheCoordinator.java index 11fc547e6b40cc..6a0442c0569f76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheCoordinator.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.proto.Types; import org.apache.doris.qe.SimpleScheduler; import org.apache.doris.system.Backend; @@ -34,8 +35,6 @@ import java.util.List; import java.util.SortedMap; import java.util.TreeMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; /** * Use consistent hashing to find the BE corresponding to the key to @@ -48,7 +47,7 @@ public class CacheCoordinator { public boolean debugModel = false; private Hashtable realNodes = new Hashtable<>(); private SortedMap virtualNodes = new TreeMap<>(); - private static Lock belock = new ReentrantLock(); + private static MonitoredReentrantLock belock = new MonitoredReentrantLock(); private long lastRefreshTime; private static CacheCoordinator cachePartition; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/help/HelpModule.java b/fe/fe-core/src/main/java/org/apache/doris/qe/help/HelpModule.java index 5c3f760bfae72b..9dafb0dd73e2bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/help/HelpModule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/help/HelpModule.java @@ -18,6 +18,7 @@ package org.apache.doris.qe.help; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantLock; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -38,7 +39,6 @@ import java.nio.file.Paths; import java.util.Enumeration; import java.util.List; -import java.util.concurrent.locks.ReentrantLock; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -88,7 +88,7 @@ public class HelpModule { private static long lastCheckTime = 0L; private boolean isloaded = false; private static String zipFilePath; - private static ReentrantLock lock = new ReentrantLock(); + private static MonitoredReentrantLock lock = new MonitoredReentrantLock(); // Files in zip is not recursive, so we only need to traverse it public void setUpByZip(String path) throws IOException, UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/TagManager.java b/fe/fe-core/src/main/java/org/apache/doris/resource/TagManager.java index 9e8181cc748c66..024b2562fe8259 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/TagManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/TagManager.java @@ -19,6 +19,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.HashMultimap; @@ -31,7 +32,6 @@ import java.io.IOException; import java.util.Map; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; /* * TagManager maintains 2 indexes: @@ -47,7 +47,7 @@ public class TagManager implements Writable { // resource id -> tag set private Map resourceIndex = Maps.newHashMap(); - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); public TagManager() { // TODO Auto-generated constructor stub diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java index ba2d2526f2e751..c3602ef9015b73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.resource.AdmissionControl; import org.apache.doris.resource.workloadgroup.QueueToken.TokenState; @@ -29,7 +30,6 @@ import java.util.LinkedList; import java.util.PriorityQueue; import java.util.Queue; -import java.util.concurrent.locks.ReentrantLock; // note(wb) refer java BlockingQueue, but support altering capacity // todo(wb) add wait time to profile @@ -37,7 +37,7 @@ public class QueryQueue { private static final Logger LOG = LogManager.getLogger(QueryQueue.class); // note(wb) used unfair by default, need more test later - private final ReentrantLock queueLock = new ReentrantLock(); + private final MonitoredReentrantLock queueLock = new MonitoredReentrantLock(); // resource group property private int maxConcurrency; private int maxQueueSize; diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java index 5ddc5fb68f8599..ab853c8138cf37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java @@ -32,6 +32,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.common.util.MasterDaemon; @@ -63,7 +64,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class WorkloadGroupMgr extends MasterDaemon implements Writable, GsonPostProcessable { @@ -90,7 +90,7 @@ public class WorkloadGroupMgr extends MasterDaemon implements Writable, GsonPost private final Map nameToWorkloadGroup = Maps.newHashMap(); private final Map idToQueryQueue = Maps.newHashMap(); private final ResourceProcNode procNode = new ResourceProcNode(); - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); @Override protected void runAfterCatalogReady() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java index b2de010b9e418f..ed1f1a41dcb94f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.plugin.audit.AuditEvent; import org.apache.doris.thrift.TQueryStatistics; @@ -35,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; // NOTE: not using a lock for beToQueryStatsMap's update because it should void global lock for all be // this may cause in some corner case missing statistics update,for example: @@ -48,7 +48,7 @@ public class WorkloadRuntimeStatusMgr extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(WorkloadRuntimeStatusMgr.class); private Map beToQueryStatsMap = Maps.newConcurrentMap(); - private final ReentrantReadWriteLock queryAuditEventLock = new ReentrantReadWriteLock(); + private final MonitoredReentrantReadWriteLock queryAuditEventLock = new MonitoredReentrantReadWriteLock(); private List queryAuditEventList = Lists.newLinkedList(); private class BeReportInfo { diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java index 3879dd83b9adfb..92940e12e055bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java @@ -26,6 +26,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.common.util.DebugUtil; @@ -62,7 +63,6 @@ import java.util.PriorityQueue; import java.util.Queue; import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class WorkloadSchedPolicyMgr extends MasterDaemon implements Writable, GsonPostProcessable { @@ -139,7 +139,7 @@ public WorkloadSchedPolicyMgr() { } } - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); public static Comparator policyComparator = new Comparator() { @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java index 97a06176fef04d..fcab2f6705526d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java @@ -21,6 +21,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Status; import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.metric.MetricRepo; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PAlterVaultSyncRequest; @@ -52,13 +53,12 @@ import java.util.concurrent.Executor; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; public class BackendServiceProxy { private static final Logger LOG = LogManager.getLogger(BackendServiceProxy.class); // use exclusive lock to make sure only one thread can add or remove client from serviceMap. // use concurrent map to allow access serviceMap in multi thread. - private ReentrantLock lock = new ReentrantLock(); + private MonitoredReentrantLock lock = new MonitoredReentrantLock(); private Executor grpcThreadPool = ThreadPoolManager.newDaemonCacheThreadPool(Config.grpc_threadmgr_threads_nums, "grpc_thread_pool", true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 56d2cb01e6a85d..3a0936f536c649 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -66,6 +66,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.annotation.LogException; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.Util; import org.apache.doris.cooldown.CooldownDelete; @@ -285,7 +286,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; // Frontend service used to serve all request for this frontend through @@ -3519,7 +3519,7 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request OlapTable olapTable = (OlapTable) table; InsertOverwriteManager overwriteManager = Env.getCurrentEnv().getInsertOverwriteManager(); - ReentrantLock taskLock = overwriteManager.getLock(taskGroupId); + MonitoredReentrantLock taskLock = overwriteManager.getLock(taskGroupId); if (taskLock == null) { errorStatus.setErrorMsgs(Lists .newArrayList(new String("cannot find task group " + taskGroupId + ", maybe already failed."))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightAuthResult.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightAuthResult.java index d18925ae6e6d52..53fb6cd713feac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightAuthResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightAuthResult.java @@ -34,7 +34,7 @@ public interface FlightAuthResult { String getRemoteIp(); static FlightAuthResult of(String userName, UserIdentity userIdentity, String remoteIp) { - return ImmutableFlightAuthResult.builder() + return org.apache.doris.service.arrowflight.auth2.ImmutableFlightAuthResult.builder() .userName(userName) .userIdentity(userIdentity) .remoteIp(remoteIp) diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/SerialExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/task/SerialExecutor.java index 22d3a3d364f7ca..0905b6dbbe0d80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/SerialExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/SerialExecutor.java @@ -17,6 +17,8 @@ package org.apache.doris.task; +import org.apache.doris.common.lock.MonitoredReentrantLock; + import com.google.common.base.Preconditions; import java.util.ArrayList; @@ -28,13 +30,12 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; public class SerialExecutor extends AbstractExecutorService { private final ExecutorService taskPool; - private final ReentrantLock lock = new ReentrantLock(); + private final MonitoredReentrantLock lock = new MonitoredReentrantLock(); private final Condition terminating = lock.newCondition(); private final BlockingQueue tasks = new LinkedBlockingQueue<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 33bbcdaaddee03..defc0c0873a574 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -43,6 +43,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.QuotaExceedException; import org.apache.doris.common.UserException; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.InternalDatabaseUtil; @@ -92,7 +93,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; /** @@ -121,7 +121,7 @@ private enum PublishResult { // the lock is used to control the access to transaction states // no other locks should be inside this lock - private final ReentrantReadWriteLock transactionLock = new ReentrantReadWriteLock(true); + private final MonitoredReentrantReadWriteLock transactionLock = new MonitoredReentrantReadWriteLock(true); // transactionId -> running TransactionState private final Map idToRunningTransactionState = Maps.newHashMap(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java index 004675413727b2..e9833336cd8d69 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.planner.StreamLoadPlanner; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.Types; @@ -62,7 +63,6 @@ import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; public class CanalSyncDataTest { private static final Logger LOG = LogManager.getLogger(CanalSyncDataTest.class); @@ -73,7 +73,7 @@ public class CanalSyncDataTest { private int batchSize = 8192; private long channelId = 100001L; - ReentrantLock getLock; + MonitoredReentrantLock getLock; CanalConnector connector; diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf index 24853b0a0c6cd2..e9448b340144ac 100644 --- a/regression-test/pipeline/p0/conf/fe.conf +++ b/regression-test/pipeline/p0/conf/fe.conf @@ -79,3 +79,6 @@ master_sync_policy = WRITE_NO_SYNC replica_sync_policy = WRITE_NO_SYNC enable_advance_next_id = true +# enable deadlock detection +enable_deadlock_detection = true +max_lock_hold_threshold_seconds = 10 From f02ef9642903df669a4c72881657a2863bc3afcd Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 8 Aug 2024 11:25:52 +0800 Subject: [PATCH 3/7] [feat](lock)add deadlock detection tool and monitored lock implementations. --- .../org/apache/doris/alter/AlterHandler.java | 4 ++-- .../apache/doris/backup/BackupHandler.java | 11 +++++----- .../apache/doris/backup/RepositoryMgr.java | 4 ++-- .../doris/binlog/BinlogConfigCache.java | 6 +++--- .../apache/doris/binlog/BinlogManager.java | 6 +++--- .../org/apache/doris/binlog/DBBinlog.java | 6 +++--- .../org/apache/doris/binlog/TableBinlog.java | 6 +++--- .../doris/blockrule/SqlBlockRuleMgr.java | 4 ++-- .../org/apache/doris/catalog/BrokerMgr.java | 4 ++-- .../doris/catalog/ColumnIdFlushDaemon.java | 4 ++-- .../java/org/apache/doris/catalog/MTMV.java | 8 ++++---- .../org/apache/doris/catalog/Resource.java | 4 ++-- .../apache/doris/catalog/StorageVault.java | 4 ++-- .../doris/cloud/catalog/CloudPartition.java | 4 ++-- .../doris/cloud/rpc/MetaServiceProxy.java | 4 ++-- .../cloud/system/CloudSystemInfoService.java | 5 +++-- .../common/lock/AbstractMonitoredLock.java | 8 ++++---- .../doris/common/lock/DeadlockMonitor.java | 20 +++---------------- .../doris/common/util/ProfileManager.java | 12 ++++++----- .../doris/common/util/RuntimeProfile.java | 14 ++++++------- .../doris/consistency/ConsistencyChecker.java | 6 +++--- .../InsertOverwriteManager.java | 10 +++++----- .../apache/doris/job/base/AbstractJob.java | 7 ++++--- .../doris/job/extensions/mtmv/MTMVJob.java | 8 ++++---- .../apache/doris/job/manager/JobManager.java | 4 ++-- .../doris/journal/bdbje/BDBEnvironment.java | 6 +++--- .../org/apache/doris/load/DeleteHandler.java | 6 +++--- .../java/org/apache/doris/load/ExportMgr.java | 4 ++-- .../main/java/org/apache/doris/load/Load.java | 6 +++--- .../doris/load/StreamLoadRecordMgr.java | 4 ++-- .../org/apache/doris/load/loadv2/LoadJob.java | 4 ++-- .../apache/doris/load/loadv2/LoadManager.java | 4 ++-- .../load/routineload/RoutineLoadJob.java | 4 ++-- .../load/routineload/RoutineLoadManager.java | 4 ++-- .../doris/load/sync/SyncJobManager.java | 6 +++--- .../sync/canal/CanalSyncDataConsumer.java | 7 +++---- .../sync/canal/CanalSyncDataReceiver.java | 6 +++--- .../load/sync/canal/SyncCanalClient.java | 6 +++--- .../mysql/authenticate/ldap/LdapManager.java | 4 ++-- .../apache/doris/mysql/privilege/Auth.java | 4 ++-- .../doris/mysql/privilege/PasswordPolicy.java | 4 ++-- .../java/org/apache/doris/policy/Policy.java | 4 ++-- .../org/apache/doris/policy/PolicyMgr.java | 4 ++-- .../java/org/apache/doris/qe/Coordinator.java | 5 +++-- .../org/apache/doris/qe/MultiLoadMgr.java | 5 +++-- .../java/org/apache/doris/qe/VariableMgr.java | 5 +++-- .../org/apache/doris/resource/TagManager.java | 4 ++-- .../resource/workloadgroup/QueryQueue.java | 4 ++-- .../workloadgroup/WorkloadGroupMgr.java | 4 ++-- .../WorkloadRuntimeStatusMgr.java | 4 ++-- .../WorkloadSchedPolicyMgr.java | 4 ++-- .../apache/doris/rpc/BackendServiceProxy.java | 4 ++-- .../doris/service/FrontendServiceImpl.java | 4 ++-- .../arrowflight/auth2/FlightAuthResult.java | 2 +- .../org/apache/doris/task/SerialExecutor.java | 5 ++--- .../load/sync/canal/CanalSyncDataTest.java | 4 ++-- 56 files changed, 153 insertions(+), 161 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java index 41c2eaf2e815d5..4c4e1118670cce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -33,7 +33,6 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.persist.RemoveAlterJobV2OperationLog; @@ -50,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; public abstract class AlterHandler extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(AlterHandler.class); @@ -64,7 +64,7 @@ public abstract class AlterHandler extends MasterDaemon { * and this requires atomic operations. So the lock must be held to do this operations. * Operations like Get or Put do not need lock. */ - protected MonitoredReentrantLock lock = new MonitoredReentrantLock(); + protected ReentrantLock lock = new ReentrantLock(); public AlterHandler(String name) { this(name, FeConstants.default_scheduler_interval_millisecond); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index ab169929a833ef..0cca0009d7e3c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -46,8 +46,6 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.Pair; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantLock; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.fs.FileSystemFactory; @@ -82,6 +80,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -94,7 +95,7 @@ public class BackupHandler extends MasterDaemon implements Writable { private RepositoryMgr repoMgr = new RepositoryMgr(); // this lock is used for updating dbIdToBackupOrRestoreJobs - private final MonitoredReentrantLock jobLock = new MonitoredReentrantLock(); + private final ReentrantLock jobLock = new ReentrantLock(); // db id -> last 10(max_backup_restore_job_num_per_db) backup/restore jobs // Newly submitted job will replace the current job, only if current job is finished or cancelled. @@ -103,7 +104,7 @@ public class BackupHandler extends MasterDaemon implements Writable { private final Map> dbIdToBackupOrRestoreJobs = new HashMap<>(); // this lock is used for handling one backup or restore request at a time. - private MonitoredReentrantLock seqlock = new MonitoredReentrantLock(); + private ReentrantLock seqlock = new ReentrantLock(); private boolean isInit = false; @@ -113,7 +114,7 @@ public class BackupHandler extends MasterDaemon implements Writable { // this map not present in persist && only in fe master memory // one table only keep one snapshot info, only keep last private final Map localSnapshots = new HashMap<>(); - private MonitoredReentrantReadWriteLock localSnapshotsLock = new MonitoredReentrantReadWriteLock(); + private ReadWriteLock localSnapshotsLock = new ReentrantReadWriteLock(); public BackupHandler() { // for persist diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java index e5ccaf1a27865b..853c1841449046 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java @@ -22,7 +22,6 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.Daemon; import org.apache.doris.fs.remote.AzureFileSystem; import org.apache.doris.fs.remote.S3FileSystem; @@ -40,6 +39,7 @@ import java.io.IOException; import java.util.List; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; /* * A manager to manage all backup repositories @@ -52,7 +52,7 @@ public class RepositoryMgr extends Daemon implements Writable, GsonPostProcessab private ConcurrentMap repoNameMap = Maps.newConcurrentMap(); private ConcurrentMap repoIdMap = Maps.newConcurrentMap(); - private MonitoredReentrantLock lock = new MonitoredReentrantLock(); + private ReentrantLock lock = new ReentrantLock(); public RepositoryMgr() { super(Repository.class.getSimpleName(), 600 * 1000 /* 10min */); diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogConfigCache.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogConfigCache.java index 0acda251c5111e..b07f5e5d87ce41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogConfigCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogConfigCache.java @@ -22,23 +22,23 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class BinlogConfigCache { private static final Logger LOG = LogManager.getLogger(BinlogConfigCache.class); private Map dbTableBinlogEnableMap; // db or table all use id - private MonitoredReentrantReadWriteLock lock; + private ReentrantReadWriteLock lock; public BinlogConfigCache() { dbTableBinlogEnableMap = new HashMap(); - lock = new MonitoredReentrantReadWriteLock(); + lock = new ReentrantReadWriteLock(); } // Get the binlog config of the specified db, return null if no such database diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java index 8d7c8cc2ad9ac9..6fc2e3f813fb07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.persist.AlterDatabasePropertyInfo; @@ -55,6 +54,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class BinlogManager { private static final int BUFFER_SIZE = 16 * 1024; @@ -67,12 +67,12 @@ public class BinlogManager { private static final Logger LOG = LogManager.getLogger(BinlogManager.class); - private MonitoredReentrantReadWriteLock lock; + private ReentrantReadWriteLock lock; private Map dbBinlogMap; private BinlogConfigCache binlogConfigCache; public BinlogManager() { - lock = new MonitoredReentrantReadWriteLock(); + lock = new ReentrantReadWriteLock(); dbBinlogMap = Maps.newHashMap(); binlogConfigCache = new BinlogConfigCache(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java index eb0243dc7ee4cf..502491004e5dab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.common.Pair; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.persist.DropPartitionInfo; import org.apache.doris.thrift.TBinlog; @@ -41,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.TreeSet; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class DBBinlog { @@ -50,7 +50,7 @@ public class DBBinlog { // The size of all binlogs. private long binlogSize; // guard for allBinlogs && tableBinlogMap - private MonitoredReentrantReadWriteLock lock; + private ReentrantReadWriteLock lock; // all binlogs contain table binlogs && create table binlog etc ... private TreeSet allBinlogs; // table binlogs @@ -70,7 +70,7 @@ public class DBBinlog { private BinlogConfigCache binlogConfigCache; public DBBinlog(BinlogConfigCache binlogConfigCache, TBinlog binlog) { - lock = new MonitoredReentrantReadWriteLock(); + lock = new ReentrantReadWriteLock(); this.dbId = binlog.getDbId(); this.binlogConfigCache = binlogConfigCache; this.binlogSize = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java index 07645b866be5fc..f3279b328c935f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.Pair; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.thrift.TBinlog; import org.apache.doris.thrift.TBinlogType; @@ -38,6 +37,7 @@ import java.util.Iterator; import java.util.List; import java.util.TreeSet; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class TableBinlog { private static final Logger LOG = LogManager.getLogger(TableBinlog.class); @@ -45,7 +45,7 @@ public class TableBinlog { private long dbId; private long tableId; private long binlogSize; - private MonitoredReentrantReadWriteLock lock; + private ReentrantReadWriteLock lock; private TreeSet binlogs; // Pair(commitSeq, timestamp), used for gc @@ -58,7 +58,7 @@ public TableBinlog(BinlogConfigCache binlogConfigCache, TBinlog binlog, long dbI this.dbId = dbId; this.tableId = tableId; this.binlogSize = 0; - lock = new MonitoredReentrantReadWriteLock(); + lock = new ReentrantReadWriteLock(); binlogs = Sets.newTreeSet(Comparator.comparingLong(TBinlog::getCommitSeq)); timestamps = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java index eb6514b16166e0..2eba1d4fae3385 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java @@ -27,7 +27,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.SqlBlockUtil; import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.gson.GsonUtils; @@ -45,6 +44,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; /** @@ -53,7 +53,7 @@ public class SqlBlockRuleMgr implements Writable { private static final Logger LOG = LogManager.getLogger(SqlBlockRuleMgr.class); - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); @SerializedName(value = "nameToSqlBlockRuleMap") private Map nameToSqlBlockRuleMap = Maps.newConcurrentMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java index 65bf667f833479..5e4e7c7cbc84de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java @@ -25,7 +25,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcNodeInterface; import org.apache.doris.common.proc.ProcResult; @@ -47,6 +46,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; /** * Broker manager @@ -64,7 +64,7 @@ public class BrokerMgr { private final Map> brokersMap = Maps.newHashMap(); // { BrokerName -> { list of FsBroker } private final Map> brokerListMap = Maps.newHashMap(); - private final MonitoredReentrantLock lock = new MonitoredReentrantLock(); + private final ReentrantLock lock = new ReentrantLock(); private BrokerProcNode procNode = null; public BrokerMgr() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnIdFlushDaemon.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnIdFlushDaemon.java index 8980158ce525ec..28007a8e666d47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnIdFlushDaemon.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnIdFlushDaemon.java @@ -19,7 +19,6 @@ import org.apache.doris.alter.AlterLightSchChangeHelper; import org.apache.doris.common.Config; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.persist.AlterLightSchemaChangeInfo; @@ -31,6 +30,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiConsumer; /** @@ -52,7 +52,7 @@ public class ColumnIdFlushDaemon extends MasterDaemon { public ColumnIdFlushDaemon() { super("colum-id-flusher", TimeUnit.HOURS.toMillis(1)); resultCollector = Maps.newHashMap(); - rwLock = new MonitoredReentrantReadWriteLock(); + rwLock = new ReentrantReadWriteLock(); if (Config.enable_convert_light_weight_schema_change) { flushFunc = this::doFlush; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 723fbe58836fb8..5097f83e2491e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -24,7 +24,6 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.job.common.TaskStatus; import org.apache.doris.job.extensions.mtmv.MTMVTask; @@ -63,11 +62,12 @@ import java.util.Map.Entry; import java.util.Optional; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class MTMV extends OlapTable { private static final Logger LOG = LogManager.getLogger(MTMV.class); - private MonitoredReentrantReadWriteLock mvRwLock; + private ReentrantReadWriteLock mvRwLock; @SerializedName("ri") private MTMVRefreshInfo refreshInfo; @@ -93,7 +93,7 @@ public class MTMV extends OlapTable { // For deserialization public MTMV() { type = TableType.MATERIALIZED_VIEW; - mvRwLock = new MonitoredReentrantReadWriteLock(true); + mvRwLock = new ReentrantReadWriteLock(true); } MTMV(MTMVParams params) { @@ -115,7 +115,7 @@ public MTMV() { this.mvPartitionInfo = params.mvPartitionInfo; this.relation = params.relation; this.refreshSnapshot = new MTMVRefreshSnapshot(); - mvRwLock = new MonitoredReentrantReadWriteLock(true); + mvRwLock = new ReentrantReadWriteLock(true); } public MTMVRefreshInfo getRefreshInfo() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index ee57b784b59f3f..0f0fd7d5de6d4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -24,7 +24,6 @@ import org.apache.doris.common.io.DeepCopy; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.persist.gson.GsonPostProcessable; @@ -41,6 +40,7 @@ import java.io.IOException; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public abstract class Resource implements Writable, GsonPostProcessable { @@ -94,7 +94,7 @@ public enum ReferenceType { @SerializedName(value = "version") protected long version = -1; - private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); public void writeLock() { lock.writeLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java index d55f7ae0c4c18a..fb68581a00e850 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java @@ -23,7 +23,6 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -34,6 +33,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.IntStream; public abstract class StorageVault { @@ -65,7 +65,7 @@ public static StorageVaultType fromString(String storageVaultTypeType) { private boolean ifNotExists; private boolean setAsDefault; - private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); public void writeLock() { lock.writeLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java index 3b1c987c0c54bb..5036a0e01c4464 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java @@ -25,7 +25,6 @@ import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; import org.apache.doris.cloud.rpc.VersionHelper; import org.apache.doris.common.Config; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.qe.ConnectContext; @@ -40,6 +39,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** @@ -54,7 +54,7 @@ public class CloudPartition extends Partition { @SerializedName(value = "tableId") private long tableId; - private MonitoredReentrantLock lock = new MonitoredReentrantLock(true); + private ReentrantLock lock = new ReentrantLock(true); public CloudPartition(long id, String name, MaterializedIndex baseIndex, DistributionInfo distributionInfo, long dbId, long tableId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index 66b0ddf611a925..5f17692180b353 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -19,7 +19,6 @@ import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.common.Config; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.rpc.RpcException; import com.google.common.collect.Maps; @@ -29,6 +28,7 @@ import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; public class MetaServiceProxy { private static final Logger LOG = LogManager.getLogger(MetaServiceProxy.class); @@ -36,7 +36,7 @@ public class MetaServiceProxy { // use exclusive lock to make sure only one thread can add or remove client from // serviceMap. // use concurrent map to allow access serviceMap in multi thread. - private MonitoredReentrantLock lock = new MonitoredReentrantLock(); + private ReentrantLock lock = new ReentrantLock(); private final Map serviceMap; static { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java index 11e0fa70de828a..48728efb003aba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java @@ -30,7 +30,6 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.ha.FrontendNodeType; import org.apache.doris.metric.MetricRepo; import org.apache.doris.qe.ConnectContext; @@ -58,13 +57,15 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class CloudSystemInfoService extends SystemInfoService { private static final Logger LOG = LogManager.getLogger(CloudSystemInfoService.class); // use rw lock to make sure only one thread can change clusterIdToBackend and clusterNameToId - private static final MonitoredReentrantReadWriteLock rwlock = new MonitoredReentrantReadWriteLock(); + private static final ReadWriteLock rwlock = new ReentrantReadWriteLock(); private static final Lock rlock = rwlock.readLock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java index 964aeafa17743a..daa8fcafdc403b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java @@ -31,7 +31,7 @@ */ public abstract class AbstractMonitoredLock { // Lock hold timeout in milliseconds - protected static final long HOLD_TIMEOUT = Config.max_lock_hold_threshold_seconds * 1000L; + protected static final long HOLD_TIMEOUT = Config.max_lock_hold_threshold_seconds * 1; private static final Logger LOG = LoggerFactory.getLogger(AbstractMonitoredLock.class); // Thread-local variable to store the lock start time @@ -58,8 +58,8 @@ protected void afterUnlock() { if (lockHoldTimeMs > HOLD_TIMEOUT) { Thread currentThread = Thread.currentThread(); String stackTrace = getThreadStackTrace(currentThread.getStackTrace()); - LOG.warn("Thread ID: {}, Thread Name: {} - Lock held for {} ms, exceeding hold timeout of {} ms \n" - + "Thread stack trace:\n{}", + LOG.warn("Thread ID: {}, Thread Name: {} - Lock held for {} ms, exceeding hold timeout of {} ms " + + "Thread stack trace:{}", currentThread.getId(), currentThread.getName(), lockHoldTimeMs, HOLD_TIMEOUT, stackTrace); } lockStartTime.remove(); @@ -99,7 +99,7 @@ private String getThreadStackTrace(StackTraceElement[] stackTrace) { for (StackTraceElement element : stackTrace) { sb.append("\tat ").append(element).append("\n"); } - return sb.toString(); + return sb.toString().replace("\n", "\\n"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java index cc6454a910e942..743404c3e50870 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java @@ -24,6 +24,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; +import java.util.Arrays; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -38,14 +39,12 @@ */ public class DeadlockMonitor { private static final Logger LOG = LoggerFactory.getLogger(DeadlockMonitor.class); - private final Gson gson; private final ThreadMXBean threadMXBean; private final ScheduledExecutorService scheduler; public DeadlockMonitor() { this.threadMXBean = ManagementFactory.getThreadMXBean(); this.scheduler = Executors.newScheduledThreadPool(1); - this.gson = new Gson(); } /** @@ -75,22 +74,9 @@ public void detectAndReportDeadlocks() { // Get information about deadlocked threads ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(deadlockedThreadIds, true, true); - - // Build a deadlock report - + String deadlockReportString = Arrays.toString(threadInfos).replace("\n", "\\n"); // Log the deadlock report - DeadlockThreadsInfo deadlockThreadsInfo = new DeadlockThreadsInfo(threadInfos); - LOG.warn("Deadlocks detected {}", gson.toJson(deadlockThreadsInfo)); + LOG.warn("Deadlocks detected {}", deadlockReportString); } - private static class DeadlockThreadsInfo { - private String[] threadInfos; - - private DeadlockThreadsInfo(ThreadInfo[] threadInfos) { - this.threadInfos = new String[threadInfos.length]; - for (int i = 0; i < threadInfos.length; i++) { - this.threadInfos[i] = threadInfos[i].toString(); - } - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java index ef8ac5235702b1..306b5f1b4e621a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java @@ -24,7 +24,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.profile.ExecutionProfile; import org.apache.doris.common.profile.MultiProfileTreeBuilder; import org.apache.doris.common.profile.Profile; @@ -61,6 +60,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; /* * if you want to visit the attribute(such as queryID,defaultDb) @@ -129,9 +131,9 @@ public void deleteFromStorage() { boolean isProfileLoaded = false; // only protect queryIdDeque; queryIdToProfileMap is concurrent, no need to protect - private MonitoredReentrantReadWriteLock lock; - private MonitoredReentrantReadWriteLock.ReadLock readLock; - private MonitoredReentrantReadWriteLock.WriteLock writeLock; + private ReentrantReadWriteLock lock; + private ReadLock readLock; + private WriteLock writeLock; // profile id is long string for brocker load // is TUniqueId for others. @@ -157,7 +159,7 @@ public static ProfileManager getInstance() { // The visiablity of ProfileManager() is package level, so that we can write ut for it. ProfileManager() { - lock = new MonitoredReentrantReadWriteLock(true); + lock = new ReentrantReadWriteLock(true); readLock = lock.readLock(); writeLock = lock.writeLock(); queryIdToProfileMap = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java index d2da55c2679523..60207b49172ba0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java @@ -20,7 +20,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.Reference; import org.apache.doris.common.io.Text; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TCounter; @@ -46,6 +45,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * It is accessed by two kinds of thread, one is to create this RuntimeProfile @@ -67,19 +67,19 @@ public class RuntimeProfile { private Map infoStrings = Maps.newHashMap(); @SerializedName(value = "infoStringsDisplayOrder") private List infoStringsDisplayOrder = Lists.newArrayList(); - private transient MonitoredReentrantReadWriteLock infoStringsLock = new MonitoredReentrantReadWriteLock(); + private transient ReentrantReadWriteLock infoStringsLock = new ReentrantReadWriteLock(); @SerializedName(value = "counterMap") private Map counterMap = Maps.newConcurrentMap(); @SerializedName(value = "childCounterMap") private Map> childCounterMap = Maps.newConcurrentMap(); // protect TreeSet in ChildCounterMap - private transient MonitoredReentrantReadWriteLock counterLock = new MonitoredReentrantReadWriteLock(); + private transient ReentrantReadWriteLock counterLock = new ReentrantReadWriteLock(); @SerializedName(value = "childMap") private Map childMap = Maps.newConcurrentMap(); @SerializedName(value = "childList") private LinkedList> childList = Lists.newLinkedList(); - private transient MonitoredReentrantReadWriteLock childLock = new MonitoredReentrantReadWriteLock(); + private transient ReentrantReadWriteLock childLock = new ReentrantReadWriteLock(); @SerializedName(value = "planNodeInfos") private List planNodeInfos = Lists.newArrayList(); @@ -126,9 +126,9 @@ public RuntimeProfile(String name, int nodeId) { } private void init() { - this.infoStringsLock = new MonitoredReentrantReadWriteLock(); - this.childLock = new MonitoredReentrantReadWriteLock(); - this.counterLock = new MonitoredReentrantReadWriteLock(); + this.infoStringsLock = new ReentrantReadWriteLock(); + this.childLock = new ReentrantReadWriteLock(); + this.counterLock = new ReentrantReadWriteLock(); } public static RuntimeProfile read(DataInput input) throws IOException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java b/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java index 73394c92946082..471e235684e390 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java @@ -28,7 +28,6 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.common.Config; import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.consistency.CheckConsistencyJob.JobState; @@ -48,6 +47,7 @@ import java.util.Map; import java.util.PriorityQueue; import java.util.Queue; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class ConsistencyChecker extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(ConsistencyChecker.class); @@ -69,7 +69,7 @@ public class ConsistencyChecker extends MasterDaemon { * * if reversal is inevitable. use db.tryLock() instead to avoid dead lock */ - private MonitoredReentrantReadWriteLock jobsLock; + private ReentrantReadWriteLock jobsLock; private int startTime; private int endTime; @@ -78,7 +78,7 @@ public ConsistencyChecker() { super("consistency checker"); jobs = Maps.newHashMap(); - jobsLock = new MonitoredReentrantReadWriteLock(); + jobsLock = new ReentrantReadWriteLock(); if (!initWorkTime()) { LOG.error("failed to init time in ConsistencyChecker. exit"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java index ccb65d3201c97e..81524ae020810e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java @@ -22,7 +22,6 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.insertoverwrite.InsertOverwriteLog.InsertOverwriteOpType; import org.apache.doris.persist.gson.GsonUtils; @@ -41,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.locks.ReentrantLock; public class InsertOverwriteManager extends MasterDaemon implements Writable { private static final Logger LOG = LogManager.getLogger(InsertOverwriteManager.class); @@ -57,7 +57,7 @@ public class InsertOverwriteManager extends MasterDaemon implements Writable { // for one task group, there may be different requests about changing a partition to new. // but we only change one time and save the relations in partitionPairs. they're protected by taskLocks @SerializedName(value = "taskLocks") - private Map taskLocks = Maps.newConcurrentMap(); + private Map taskLocks = Maps.newConcurrentMap(); // >. no need concern which task it belongs to. @SerializedName(value = "partitionPairs") private Map> partitionPairs = Maps.newConcurrentMap(); @@ -94,7 +94,7 @@ public long registerTask(long dbId, long tableId, List tempPartitionName public long registerTaskGroup() { long groupId = Env.getCurrentEnv().getNextId(); taskGroups.put(groupId, new ArrayList()); - taskLocks.put(groupId, new MonitoredReentrantLock()); + taskLocks.put(groupId, new ReentrantLock()); partitionPairs.put(groupId, Maps.newConcurrentMap()); return groupId; } @@ -141,7 +141,7 @@ public void recordPartitionPairs(long groupId, List oldIds, List new } // lock is a symbol of TaskGroup exist. if not, means already failed. - public MonitoredReentrantLock getLock(long groupId) { + public ReentrantLock getLock(long groupId) { return taskLocks.get(groupId); } @@ -149,7 +149,7 @@ public MonitoredReentrantLock getLock(long groupId) { // it will cause ConcurrentModification or NullPointer. public void taskGroupFail(long groupId) { LOG.info("insert overwrite auto detect partition task group [" + groupId + "] failed"); - MonitoredReentrantLock lock = getLock(groupId); + ReentrantLock lock = getLock(groupId); lock.lock(); try { // will rollback temp partitions in `taskFail` diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java index 98291cdc6fa742..3f595d6daf5362 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java @@ -23,7 +23,6 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.common.util.TimeUtils; @@ -53,6 +52,8 @@ import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @Data @@ -145,7 +146,7 @@ public AbstractJob(Long jobId, String jobName, JobStatus jobStatus, private CopyOnWriteArrayList runningTasks = new CopyOnWriteArrayList<>(); - private MonitoredReentrantLock createTaskLock = new MonitoredReentrantLock(); + private Lock createTaskLock = new ReentrantLock(); @Override public void cancelAllTasks() throws JobException { @@ -287,7 +288,7 @@ public static AbstractJob readFields(DataInput in) throws IOException { String jsonJob = Text.readString(in); AbstractJob job = GsonUtils.GSON.fromJson(jsonJob, AbstractJob.class); job.runningTasks = new CopyOnWriteArrayList(); - job.createTaskLock = new MonitoredReentrantLock(); + job.createTaskLock = new ReentrantLock(); return job; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java index a4e1c8e9867a7b..5d7cf4435b9c7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java @@ -28,7 +28,6 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.job.base.AbstractJob; @@ -52,10 +51,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class MTMVJob extends AbstractJob { private static final Logger LOG = LogManager.getLogger(MTMVJob.class); - private MonitoredReentrantReadWriteLock jobRwLock; + private ReentrantReadWriteLock jobRwLock; private static final ShowResultSetMetaData JOB_META_DATA = ShowResultSetMetaData.builder() @@ -108,14 +108,14 @@ public class MTMVJob extends AbstractJob { private long mtmvId; public MTMVJob() { - jobRwLock = new MonitoredReentrantReadWriteLock(true); + jobRwLock = new ReentrantReadWriteLock(true); } public MTMVJob(long dbId, long mtmvId) { this.dbId = dbId; this.mtmvId = mtmvId; super.setCreateTimeMs(System.currentTimeMillis()); - jobRwLock = new MonitoredReentrantReadWriteLock(true); + jobRwLock = new ReentrantReadWriteLock(true); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java index 7b9d849f3b9bf4..39646bab18f785 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java @@ -29,7 +29,6 @@ import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.datasource.InternalCatalog; @@ -56,6 +55,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @Log4j2 @@ -67,7 +67,7 @@ public class JobManager, C> implements Writable { // lock for job // lock is private and must use after db lock - private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java index 55a1ff3983bdbb..ff162b0c9fa522 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java @@ -19,7 +19,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.ha.BDBHA; import org.apache.doris.ha.BDBStateChangeListener; import org.apache.doris.ha.FrontendNodeType; @@ -59,6 +58,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; import java.util.stream.Collectors; @@ -79,7 +79,7 @@ public class BDBEnvironment { private ReplicationConfig replicationConfig; private DatabaseConfig dbConfig; private Database epochDB = null; // used for fencing - private MonitoredReentrantReadWriteLock lock; + private ReentrantReadWriteLock lock; private List openedDatabases; private final boolean isElectable; @@ -87,7 +87,7 @@ public class BDBEnvironment { public BDBEnvironment(boolean isElectable, boolean metadataFailureRecovery) { openedDatabases = new ArrayList(); - this.lock = new MonitoredReentrantReadWriteLock(true); + this.lock = new ReentrantReadWriteLock(true); this.isElectable = isElectable; this.metadataFailureRecovery = metadataFailureRecovery; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 1d96eea960fbec..299dac295dc909 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -27,7 +27,6 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; @@ -55,6 +54,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.UUID; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class DeleteHandler implements Writable { private static final Logger LOG = LogManager.getLogger(DeleteHandler.class); @@ -66,12 +66,12 @@ public class DeleteHandler implements Writable { @SerializedName(value = "dbToDeleteInfos") private final Map> dbToDeleteInfos; - private final MonitoredReentrantReadWriteLock lock; + private final ReentrantReadWriteLock lock; public DeleteHandler() { idToDeleteJob = Maps.newConcurrentMap(); dbToDeleteInfos = Maps.newConcurrentMap(); - lock = new MonitoredReentrantReadWriteLock(); + lock = new ReentrantReadWriteLock(); } public void readLock() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java index 85839c8716a30d..bce0c5c4d9f195 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java @@ -32,7 +32,6 @@ import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.OrderByPair; import org.apache.doris.common.util.TimeUtils; @@ -57,6 +56,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -68,7 +68,7 @@ public class ExportMgr { // lock for export job // lock is private and must use after db lock - private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); public ExportMgr() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index cdbbf1628107be..0323f3ea62ee15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -62,7 +62,6 @@ import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.ExprUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; @@ -94,6 +93,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class Load { private static final Logger LOG = LogManager.getLogger(Load.class); @@ -120,7 +120,7 @@ public class Load { // lock for load job // lock is private and must use after db lock - private MonitoredReentrantReadWriteLock lock; + private ReentrantReadWriteLock lock; static { Set pendingDestStates = Sets.newHashSet(); @@ -180,7 +180,7 @@ public Load() { idToQuorumFinishedLoadJob = Maps.newLinkedHashMap(); loadingPartitionIds = Sets.newHashSet(); dbToMiniLabels = Maps.newHashMap(); - lock = new MonitoredReentrantReadWriteLock(true); + lock = new ReentrantReadWriteLock(true); } public void readLock() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java index 0a60bcbad41422..1e1dc192ceeac1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java @@ -26,7 +26,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; @@ -62,6 +61,7 @@ import java.util.PriorityQueue; import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class StreamLoadRecordMgr extends MasterDaemon { @@ -107,7 +107,7 @@ public int compare(StreamLoadItem s1, StreamLoadItem s2) { Queue streamLoadRecordHeap = new PriorityQueue<>(new StreamLoadComparator()); private Map> dbIdToLabelToStreamLoadRecord = Maps.newConcurrentMap(); - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); public StreamLoadRecordMgr(String name, long intervalMs) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index bad440b7c1f2b0..3268fb3c464836 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -39,7 +39,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.common.util.TimeUtils; @@ -82,6 +81,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; public abstract class LoadJob extends AbstractTxnStateChangeCallback implements LoadTaskCallback, Writable, GsonPostProcessable { @@ -134,7 +134,7 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback // During committing, the load job could not be cancelled. protected boolean isCommitting = false; - protected MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); // this request id is only used for checking if a load begin request is a duplicate request. protected TUniqueId requestId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java index e87ca2bf8b4902..1026689d8f1448 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java @@ -40,7 +40,6 @@ import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.datasource.InternalCatalog; @@ -84,6 +83,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -101,7 +101,7 @@ public class LoadManager implements Writable { protected Map>> dbIdToLabelToLoadJobs = Maps.newConcurrentMap(); protected LoadJobScheduler loadJobScheduler; - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); private MysqlLoadManager mysqlLoadManager; private TokenManager tokenManager; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index ef0cc2a3111dc9..a25cd99985892e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -42,7 +42,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; @@ -96,6 +95,7 @@ import java.util.Optional; import java.util.Queue; import java.util.UUID; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Routine load job is a function which stream load data from streaming medium to doris. @@ -276,7 +276,7 @@ public boolean isFinalState() { @SerializedName("cm") protected String comment = ""; - protected MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete protected Expr deleteCondition; // TODO(ml): error sample diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index 234c1228140fb8..8b0ebab3fbe7d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -40,7 +40,6 @@ import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.datasource.InternalCatalog; @@ -74,6 +73,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -89,7 +89,7 @@ public class RoutineLoadManager implements Writable { private ConcurrentHashMap multiLoadTaskTxnIdToRoutineLoadJobId = new ConcurrentHashMap<>(); - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/SyncJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/SyncJobManager.java index 1d4c27f3c7cdbe..9455d2c9ea623a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/SyncJobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/SyncJobManager.java @@ -27,7 +27,6 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.load.sync.canal.CanalDestination; @@ -50,6 +49,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -60,12 +60,12 @@ public class SyncJobManager implements Writable { private Map>> dbIdToJobNameToSyncJobs; - private MonitoredReentrantReadWriteLock lock; + private ReentrantReadWriteLock lock; public SyncJobManager() { idToSyncJob = Maps.newConcurrentMap(); dbIdToJobNameToSyncJobs = Collections.synchronizedMap(Maps.newLinkedHashMap()); - lock = new MonitoredReentrantReadWriteLock(true); + lock = new ReentrantReadWriteLock(true); } public void addDataSyncJob(CreateDataSyncJobStmt stmt) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java index 32af4afc33f3d7..25500772248992 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java @@ -20,7 +20,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Status; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.load.sync.SyncChannelHandle; import org.apache.doris.load.sync.SyncDataConsumer; import org.apache.doris.load.sync.SyncFailMsg; @@ -54,6 +53,7 @@ import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; public class CanalSyncDataConsumer extends SyncDataConsumer { private static Logger logger = LogManager.getLogger(CanalSyncDataConsumer.class); @@ -70,7 +70,7 @@ public class CanalSyncDataConsumer extends SyncDataConsumer { private PositionMeta positionMeta; private LinkedBlockingQueue> dataBlockingQueue; private SyncChannelHandle handle; - private MonitoredReentrantLock getLock; + private ReentrantLock getLock; private int sleepTimeMs; private long commitIntervalSecond; @@ -82,8 +82,7 @@ public void setChannels(Map idToChannels) { this.idToChannels = idToChannels; } - public CanalSyncDataConsumer(CanalSyncJob syncJob, CanalConnector connector, - MonitoredReentrantLock getLock, boolean debug) { + public CanalSyncDataConsumer(CanalSyncJob syncJob, CanalConnector connector, ReentrantLock getLock, boolean debug) { super(debug); this.syncJob = syncJob; this.connector = connector; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataReceiver.java index f894bae7a4aa62..c3ebcbc4928739 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataReceiver.java @@ -17,7 +17,6 @@ package org.apache.doris.load.sync.canal; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.load.sync.SyncDataReceiver; import com.alibaba.otter.canal.client.CanalConnector; @@ -26,20 +25,21 @@ import org.apache.logging.log4j.Logger; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; public class CanalSyncDataReceiver extends SyncDataReceiver { private static Logger LOG = LogManager.getLogger(CanalSyncDataReceiver.class); private CanalSyncJob syncJob; private CanalConnector connector; - private MonitoredReentrantLock getLock; + private ReentrantLock getLock; private CanalSyncDataConsumer consumer; private String destination; private String filter; private long sleepTimeMs; public CanalSyncDataReceiver(CanalSyncJob syncJob, CanalConnector connector, String destination, String filter, - CanalSyncDataConsumer consumer, int readBatchSize, MonitoredReentrantLock getLock) { + CanalSyncDataConsumer consumer, int readBatchSize, ReentrantLock getLock) { super(readBatchSize); this.syncJob = syncJob; this.connector = connector; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java index 847f24e0b30cc9..d41def0465a5c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java @@ -17,7 +17,6 @@ package org.apache.doris.load.sync.canal; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.load.sync.SyncChannel; import com.alibaba.otter.canal.client.CanalConnector; @@ -28,6 +27,7 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; public class SyncCanalClient { protected static Logger logger = LogManager.getLogger(SyncCanalClient.class); @@ -39,8 +39,8 @@ public class SyncCanalClient { // channel id -> channel private final Map idToChannels; - protected MonitoredReentrantLock lock = new MonitoredReentrantLock(true); - protected MonitoredReentrantLock getLock = new MonitoredReentrantLock(); + protected ReentrantLock lock = new ReentrantLock(true); + protected ReentrantLock getLock = new ReentrantLock(); protected void lock() { lock.lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapManager.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapManager.java index e8ebccc46e0008..5f6003cd6c146c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapManager.java @@ -25,7 +25,6 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.LdapConfig; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.mysql.authenticate.AuthenticateType; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.PrivBitSet; @@ -43,6 +42,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Encapsulates LDAP service interfaces and caches user LDAP information. @@ -56,7 +56,7 @@ public class LdapManager { private final Map ldapUserInfoCache = Maps.newHashMap(); - private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java index 9e1b7197c28996..2073e33cb4fdfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java @@ -54,7 +54,6 @@ import org.apache.doris.common.PatternMatcherException; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.MysqlPassword; import org.apache.doris.mysql.authenticate.AuthenticateType; @@ -88,6 +87,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @@ -116,7 +116,7 @@ public class Auth implements Writable { private PasswordPolicyManager passwdPolicyManager = new PasswordPolicyManager(); - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); private void readLock() { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PasswordPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PasswordPolicy.java index 080d413c051157..367eec89827274 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PasswordPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PasswordPolicy.java @@ -23,7 +23,6 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.persist.gson.GsonUtils; @@ -44,6 +43,7 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Password policy for a specific user. @@ -56,7 +56,7 @@ public class PasswordPolicy implements Writable { private static final Logger LOG = LogManager.getLogger(PasswordPolicy.class); - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); private static final String EXPIRATION_SECONDS = "password_policy.expiration_seconds"; private static final String PASSWORD_CREATION_TIME = "password_policy.password_creation_time"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java index c99158cfcf3a35..01c5399d4ab974 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java @@ -24,7 +24,6 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -39,6 +38,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Base class for Policy. @@ -60,7 +60,7 @@ public abstract class Policy implements Writable, GsonPostProcessable { @SerializedName(value = "version") protected long version = -1; - private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); public void writeLock() { lock.writeLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java index 6836bd694e3b0c..6e8bd4f08cb2f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java @@ -35,7 +35,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ShowResultSet; @@ -64,6 +63,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; /** @@ -72,7 +72,7 @@ public class PolicyMgr implements Writable { private static final Logger LOG = LogManager.getLogger(PolicyMgr.class); - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); @SerializedName(value = "typeToPolicyMap") private Map> typeToPolicyMap = Maps.newConcurrentMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 16caea9f94bbc9..4ee46d3bec3c38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -29,7 +29,6 @@ import org.apache.doris.common.Status; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.profile.ExecutionProfile; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.common.util.DebugUtil; @@ -162,6 +161,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -205,7 +206,7 @@ public class Coordinator implements CoordInterface { private TNetworkAddress currentConnectFE; // protects all fields below - private final MonitoredReentrantLock lock = new MonitoredReentrantLock(); + private final Lock lock = new ReentrantLock(); // If true, the query is done returning all results. It is possible that the // coordinator still needs to wait for cleanup on remote fragments (e.g. queries diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MultiLoadMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MultiLoadMgr.java index c182b7f10814b7..63f6de760cb102 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MultiLoadMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MultiLoadMgr.java @@ -35,7 +35,6 @@ import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.load.EtlJobType; import org.apache.doris.load.loadv2.JobState; @@ -61,6 +60,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; // Class used to record state of multi-load operation @@ -68,7 +69,7 @@ public class MultiLoadMgr { private static final Logger LOG = LogManager.getLogger(MultiLoadMgr.class); private Map infoMap = Maps.newHashMap(); - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(true); + private ReadWriteLock lock = new ReentrantReadWriteLock(true); // Start multi-load transaction. // Label is the only need parameter, maybe other properties? diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 3461061b56f9d0..1b1a317f5f0bb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -31,7 +31,6 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.VariableAnnotation; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.persist.GlobalVarPersistInfo; @@ -64,6 +63,8 @@ import java.util.Map; import java.util.StringJoiner; import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.annotation.Nullable; /** @@ -137,7 +138,7 @@ public class VariableMgr { private static ImmutableMap ctxByVarNameForCkpt; // Global read/write lock to protect access of globalSessionVariable. - private static final MonitoredReentrantReadWriteLock rwlock = new MonitoredReentrantReadWriteLock(); + private static final ReadWriteLock rwlock = new ReentrantReadWriteLock(); private static final Lock rlock = rwlock.readLock(); private static final Lock wlock = rwlock.writeLock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/TagManager.java b/fe/fe-core/src/main/java/org/apache/doris/resource/TagManager.java index 024b2562fe8259..9e8181cc748c66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/TagManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/TagManager.java @@ -19,7 +19,6 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.HashMultimap; @@ -32,6 +31,7 @@ import java.io.IOException; import java.util.Map; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; /* * TagManager maintains 2 indexes: @@ -47,7 +47,7 @@ public class TagManager implements Writable { // resource id -> tag set private Map resourceIndex = Maps.newHashMap(); - private MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); public TagManager() { // TODO Auto-generated constructor stub diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java index c3602ef9015b73..ba2d2526f2e751 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java @@ -20,7 +20,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.resource.AdmissionControl; import org.apache.doris.resource.workloadgroup.QueueToken.TokenState; @@ -30,6 +29,7 @@ import java.util.LinkedList; import java.util.PriorityQueue; import java.util.Queue; +import java.util.concurrent.locks.ReentrantLock; // note(wb) refer java BlockingQueue, but support altering capacity // todo(wb) add wait time to profile @@ -37,7 +37,7 @@ public class QueryQueue { private static final Logger LOG = LogManager.getLogger(QueryQueue.class); // note(wb) used unfair by default, need more test later - private final MonitoredReentrantLock queueLock = new MonitoredReentrantLock(); + private final ReentrantLock queueLock = new ReentrantLock(); // resource group property private int maxConcurrency; private int maxQueueSize; diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java index ab853c8138cf37..5ddc5fb68f8599 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java @@ -32,7 +32,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.common.util.MasterDaemon; @@ -64,6 +63,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class WorkloadGroupMgr extends MasterDaemon implements Writable, GsonPostProcessable { @@ -90,7 +90,7 @@ public class WorkloadGroupMgr extends MasterDaemon implements Writable, GsonPost private final Map nameToWorkloadGroup = Maps.newHashMap(); private final Map idToQueryQueue = Maps.newHashMap(); private final ResourceProcNode procNode = new ResourceProcNode(); - private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); @Override protected void runAfterCatalogReady() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java index ed1f1a41dcb94f..b2de010b9e418f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java @@ -20,7 +20,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.plugin.audit.AuditEvent; import org.apache.doris.thrift.TQueryStatistics; @@ -36,6 +35,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; // NOTE: not using a lock for beToQueryStatsMap's update because it should void global lock for all be // this may cause in some corner case missing statistics update,for example: @@ -48,7 +48,7 @@ public class WorkloadRuntimeStatusMgr extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(WorkloadRuntimeStatusMgr.class); private Map beToQueryStatsMap = Maps.newConcurrentMap(); - private final MonitoredReentrantReadWriteLock queryAuditEventLock = new MonitoredReentrantReadWriteLock(); + private final ReentrantReadWriteLock queryAuditEventLock = new ReentrantReadWriteLock(); private List queryAuditEventList = Lists.newLinkedList(); private class BeReportInfo { diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java index 92940e12e055bf..3879dd83b9adfb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java @@ -26,7 +26,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.common.util.DebugUtil; @@ -63,6 +62,7 @@ import java.util.PriorityQueue; import java.util.Queue; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class WorkloadSchedPolicyMgr extends MasterDaemon implements Writable, GsonPostProcessable { @@ -139,7 +139,7 @@ public WorkloadSchedPolicyMgr() { } } - private final MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock(); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); public static Comparator policyComparator = new Comparator() { @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java index fcab2f6705526d..97a06176fef04d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java @@ -21,7 +21,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Status; import org.apache.doris.common.ThreadPoolManager; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.metric.MetricRepo; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PAlterVaultSyncRequest; @@ -53,12 +52,13 @@ import java.util.concurrent.Executor; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; public class BackendServiceProxy { private static final Logger LOG = LogManager.getLogger(BackendServiceProxy.class); // use exclusive lock to make sure only one thread can add or remove client from serviceMap. // use concurrent map to allow access serviceMap in multi thread. - private MonitoredReentrantLock lock = new MonitoredReentrantLock(); + private ReentrantLock lock = new ReentrantLock(); private Executor grpcThreadPool = ThreadPoolManager.newDaemonCacheThreadPool(Config.grpc_threadmgr_threads_nums, "grpc_thread_pool", true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 3a0936f536c649..56d2cb01e6a85d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -66,7 +66,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.annotation.LogException; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.Util; import org.apache.doris.cooldown.CooldownDelete; @@ -286,6 +285,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; // Frontend service used to serve all request for this frontend through @@ -3519,7 +3519,7 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request OlapTable olapTable = (OlapTable) table; InsertOverwriteManager overwriteManager = Env.getCurrentEnv().getInsertOverwriteManager(); - MonitoredReentrantLock taskLock = overwriteManager.getLock(taskGroupId); + ReentrantLock taskLock = overwriteManager.getLock(taskGroupId); if (taskLock == null) { errorStatus.setErrorMsgs(Lists .newArrayList(new String("cannot find task group " + taskGroupId + ", maybe already failed."))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightAuthResult.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightAuthResult.java index 53fb6cd713feac..d18925ae6e6d52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightAuthResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightAuthResult.java @@ -34,7 +34,7 @@ public interface FlightAuthResult { String getRemoteIp(); static FlightAuthResult of(String userName, UserIdentity userIdentity, String remoteIp) { - return org.apache.doris.service.arrowflight.auth2.ImmutableFlightAuthResult.builder() + return ImmutableFlightAuthResult.builder() .userName(userName) .userIdentity(userIdentity) .remoteIp(remoteIp) diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/SerialExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/task/SerialExecutor.java index 0905b6dbbe0d80..22d3a3d364f7ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/SerialExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/SerialExecutor.java @@ -17,8 +17,6 @@ package org.apache.doris.task; -import org.apache.doris.common.lock.MonitoredReentrantLock; - import com.google.common.base.Preconditions; import java.util.ArrayList; @@ -30,12 +28,13 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; public class SerialExecutor extends AbstractExecutorService { private final ExecutorService taskPool; - private final MonitoredReentrantLock lock = new MonitoredReentrantLock(); + private final ReentrantLock lock = new ReentrantLock(); private final Condition terminating = lock.newCondition(); private final BlockingQueue tasks = new LinkedBlockingQueue<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java index e9833336cd8d69..004675413727b2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java @@ -23,7 +23,6 @@ import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; -import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.planner.StreamLoadPlanner; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.Types; @@ -63,6 +62,7 @@ import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; public class CanalSyncDataTest { private static final Logger LOG = LogManager.getLogger(CanalSyncDataTest.class); @@ -73,7 +73,7 @@ public class CanalSyncDataTest { private int batchSize = 8192; private long channelId = 100001L; - MonitoredReentrantLock getLock; + ReentrantLock getLock; CanalConnector connector; From e5db911864f95053fa48436ba7a34e77e25ae0e9 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 8 Aug 2024 11:28:07 +0800 Subject: [PATCH 4/7] [feat](lock)add deadlock detection tool and monitored lock implementations. --- .../src/main/java/org/apache/doris/qe/help/HelpModule.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/help/HelpModule.java b/fe/fe-core/src/main/java/org/apache/doris/qe/help/HelpModule.java index 9dafb0dd73e2bf..5c3f760bfae72b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/help/HelpModule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/help/HelpModule.java @@ -18,7 +18,6 @@ package org.apache.doris.qe.help; import org.apache.doris.common.UserException; -import org.apache.doris.common.lock.MonitoredReentrantLock; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -39,6 +38,7 @@ import java.nio.file.Paths; import java.util.Enumeration; import java.util.List; +import java.util.concurrent.locks.ReentrantLock; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -88,7 +88,7 @@ public class HelpModule { private static long lastCheckTime = 0L; private boolean isloaded = false; private static String zipFilePath; - private static MonitoredReentrantLock lock = new MonitoredReentrantLock(); + private static ReentrantLock lock = new ReentrantLock(); // Files in zip is not recursive, so we only need to traverse it public void setUpByZip(String path) throws IOException, UserException { From 9a7d28c84bd08fe223c2875606ef04ee21b964b0 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 8 Aug 2024 11:30:24 +0800 Subject: [PATCH 5/7] fix checkstyle --- .../main/java/org/apache/doris/common/lock/DeadlockMonitor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java index 743404c3e50870..4fcda97dbd1ad5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/DeadlockMonitor.java @@ -17,7 +17,6 @@ package org.apache.doris.common.lock; -import com.google.gson.Gson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From b11ac6aeb3e60cdaee4696e8cad9baa5d8de13ac Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 8 Aug 2024 14:27:56 +0800 Subject: [PATCH 6/7] fix checkstyle --- .../java/org/apache/doris/common/Config.java | 4 +- .../org/apache/doris/catalog/Database.java | 6 +-- .../java/org/apache/doris/catalog/Env.java | 6 +-- .../java/org/apache/doris/catalog/Table.java | 8 ++-- .../common/lock/AbstractMonitoredLock.java | 2 +- .../common/lock/MonitoredReentrantLock.java | 5 +++ .../lock/MonitoredReentrantReadWriteLock.java | 26 +++++++----- .../common/util/QueryableReentrantLock.java | 41 ------------------- .../util/QueryableReentrantReadWriteLock.java | 41 ------------------- .../doris/datasource/InternalCatalog.java | 4 +- .../doris/mysql/privilege/UserManager.java | 4 +- .../util/QueryableReentrantLockTest.java | 4 +- 12 files changed, 41 insertions(+), 110 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantLock.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantReadWriteLock.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index a39e0e5c89cef3..bd68a30a23ddc6 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2997,7 +2997,7 @@ public static int metaServiceRpcRetryTimes() { "Deadlock detection interval time, unit minute"}) public static long deadlock_detection_interval_minute = 5; - @ConfField(description = {"表示最大锁持有时间，超过该时间会打印告警日志，单位秒", + @ConfField(mutable = true, description = {"表示最大锁持有时间，超过该时间会打印告警日志，单位秒", "Maximum lock hold time; logs a warning if exceeded"}) - public static long max_lock_hold_threshold_seconds = 30; + public static long max_lock_hold_threshold_seconds = 10; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index 5637e27e0d765b..6862c3b61c02a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -29,9 +29,9 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.common.util.QueryableReentrantReadWriteLock; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.persist.CreateTableInfo; @@ -87,7 +87,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf, @SerializedName(value = "fullQualifiedName") private volatile String fullQualifiedName; - private QueryableReentrantReadWriteLock rwLock; + private MonitoredReentrantReadWriteLock rwLock; // table family group map private final Map idToTable; @@ -138,7 +138,7 @@ public Database(long id, String name) { if (this.fullQualifiedName == null) { this.fullQualifiedName = ""; } - this.rwLock = new QueryableReentrantReadWriteLock(true); + this.rwLock = new MonitoredReentrantReadWriteLock(true); this.idToTable = Maps.newConcurrentMap(); this.nameToTable = Maps.newConcurrentMap(); this.lowerCaseToTableName = Maps.newConcurrentMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 8323feca4ce073..33f71576eab13e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -111,6 +111,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.CountingDataOutputStream; import org.apache.doris.common.io.Text; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.publish.TopicPublisher; import org.apache.doris.common.publish.TopicPublisherThread; import org.apache.doris.common.publish.WorkloadGroupPublisher; @@ -122,7 +123,6 @@ import org.apache.doris.common.util.NetUtils; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.common.util.QueryableReentrantLock; import org.apache.doris.common.util.SmallFileMgr; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; @@ -362,7 +362,7 @@ public class Env { // We use fair ReentrantLock to avoid starvation. Do not use this lock in critical code pass // because fair lock has poor performance. // Using QueryableReentrantLock to print owner thread in debug mode. - private QueryableReentrantLock lock; + private MonitoredReentrantLock lock; private CatalogMgr catalogMgr; private GlobalFunctionMgr globalFunctionMgr; @@ -680,7 +680,7 @@ public Env(boolean isCheckpointCatalog) { this.syncJobManager = new SyncJobManager(); this.alter = new Alter(); this.consistencyChecker = new ConsistencyChecker(); - this.lock = new QueryableReentrantLock(true); + this.lock = new MonitoredReentrantLock(true); this.backupHandler = new BackupHandler(this); this.metaDir = Config.meta_dir; this.publishVersionDaemon = new PublishVersionDaemon(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 59379f00baad0f..0665b128bfdc87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -28,7 +28,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.lock.MonitoredReentrantLock; -import org.apache.doris.common.util.QueryableReentrantReadWriteLock; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.common.util.Util; import org.apache.doris.persist.gson.GsonPostProcessable; @@ -84,7 +84,7 @@ public abstract class Table extends MetaObject implements Writable, TableIf, Gso protected TableType type; @SerializedName(value = "createTime") protected long createTime; - protected QueryableReentrantReadWriteLock rwLock; + protected MonitoredReentrantReadWriteLock rwLock; // Used for queuing commit transactifon tasks to avoid fdb transaction conflicts, // especially to reduce conflicts when obtaining delete bitmap update locks for // MoW table @@ -134,7 +134,7 @@ public Table(TableType type) { this.type = type; this.fullSchema = Lists.newArrayList(); this.nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - this.rwLock = new QueryableReentrantReadWriteLock(true); + this.rwLock = new MonitoredReentrantReadWriteLock(true); if (Config.check_table_lock_leaky) { this.readLockThreads = Maps.newConcurrentMap(); } @@ -158,7 +158,7 @@ public Table(long id, String tableName, TableType type, List fullSchema) // Only view in with-clause have null base Preconditions.checkArgument(type == TableType.VIEW, "Table has no columns"); } - this.rwLock = new QueryableReentrantReadWriteLock(true); + this.rwLock = new MonitoredReentrantReadWriteLock(true); this.createTime = Instant.now().getEpochSecond(); if (Config.check_table_lock_leaky) { this.readLockThreads = Maps.newConcurrentMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java index daa8fcafdc403b..fbe1a10e8d63a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java @@ -31,7 +31,7 @@ */ public abstract class AbstractMonitoredLock { // Lock hold timeout in milliseconds - protected static final long HOLD_TIMEOUT = Config.max_lock_hold_threshold_seconds * 1; + protected static final long HOLD_TIMEOUT = Config.max_lock_hold_threshold_seconds * 1000; private static final Logger LOG = LoggerFactory.getLogger(AbstractMonitoredLock.class); // Thread-local variable to store the lock start time diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java index 263f37bca53f5f..60211a6a8a8c9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantLock.java @@ -90,4 +90,9 @@ public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException lockMonitor.afterTryLock(acquired, start); // Log result and elapsed time return acquired; } + + @Override + public Thread getOwner() { + return super.getOwner(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java index 68f48eca93a8d5..7a6f0db5938b23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java @@ -24,10 +24,9 @@ * monitoring capabilities for read and write locks. */ public class MonitoredReentrantReadWriteLock extends ReentrantReadWriteLock { - private final AbstractMonitoredLock readLockMonitor = new AbstractMonitoredLock() { - }; - private final AbstractMonitoredLock writeLockMonitor = new AbstractMonitoredLock() { - }; + // Monitored read and write lock instances + private final ReadLock readLock = new ReadLock(this); + private final WriteLock writeLock = new WriteLock(this); // Constructor for creating a monitored lock with fairness option public MonitoredReentrantReadWriteLock(boolean fair) { @@ -42,6 +41,7 @@ public MonitoredReentrantReadWriteLock() { */ public class ReadLock extends ReentrantReadWriteLock.ReadLock { private static final long serialVersionUID = 1L; + private final AbstractMonitoredLock monitor = new AbstractMonitoredLock() {}; /** * Constructs a new ReadLock instance. @@ -59,7 +59,7 @@ protected ReadLock(ReentrantReadWriteLock lock) { @Override public void lock() { super.lock(); - readLockMonitor.afterLock(); + monitor.afterLock(); } /** @@ -68,7 +68,7 @@ public void lock() { */ @Override public void unlock() { - readLockMonitor.afterUnlock(); + monitor.afterUnlock(); super.unlock(); } } @@ -78,6 +78,7 @@ public void unlock() { */ public class WriteLock extends ReentrantReadWriteLock.WriteLock { private static final long serialVersionUID = 1L; + private final AbstractMonitoredLock monitor = new AbstractMonitoredLock() {}; /** * Constructs a new WriteLock instance. @@ -95,7 +96,7 @@ protected WriteLock(ReentrantReadWriteLock lock) { @Override public void lock() { super.lock(); - writeLockMonitor.afterLock(); + monitor.afterLock(); } /** @@ -104,7 +105,7 @@ public void lock() { */ @Override public void unlock() { - writeLockMonitor.afterUnlock(); + monitor.afterUnlock(); super.unlock(); } } @@ -116,7 +117,7 @@ public void unlock() { */ @Override public ReadLock readLock() { - return new ReadLock(this); + return readLock; } /** @@ -126,6 +127,11 @@ public ReadLock readLock() { */ @Override public WriteLock writeLock() { - return new WriteLock(this); + return writeLock; + } + + @Override + public Thread getOwner() { + return super.getOwner(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantLock.java deleted file mode 100644 index 1f0283434f99a0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantLock.java +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.util; - -import java.util.concurrent.locks.ReentrantLock; - -/* - * This Lock is for exposing the getOwner() method, - * which is a protected method of ReentrantLock - */ -public class QueryableReentrantLock extends ReentrantLock { - private static final long serialVersionUID = 1L; - - public QueryableReentrantLock() { - super(); - } - - public QueryableReentrantLock(boolean fair) { - super(fair); - } - - @Override - public Thread getOwner() { - return super.getOwner(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantReadWriteLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantReadWriteLock.java deleted file mode 100644 index 3f55b54229710f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/QueryableReentrantReadWriteLock.java +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.util; - -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/* - * This Lock is for exposing the getOwner() method, - * which is a protected method of ReentrantLock - */ -public class QueryableReentrantReadWriteLock extends ReentrantReadWriteLock { - private static final long serialVersionUID = 1L; - - public QueryableReentrantReadWriteLock() { - super(); - } - - public QueryableReentrantReadWriteLock(boolean fair) { - super(fair); - } - - @Override - public Thread getOwner() { - return super.getOwner(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 38cde9892011db..9ff1ba9272b54d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -127,13 +127,13 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.io.CountingDataOutputStream; +import org.apache.doris.common.lock.MonitoredReentrantLock; import org.apache.doris.common.util.DbUtil; import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.IdGeneratorUtil; import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.common.util.QueryableReentrantLock; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; @@ -216,7 +216,7 @@ public class InternalCatalog implements CatalogIf { private static final Logger LOG = LogManager.getLogger(InternalCatalog.class); - private QueryableReentrantLock lock = new QueryableReentrantLock(true); + private MonitoredReentrantLock lock = new MonitoredReentrantLock(true); private transient ConcurrentHashMap idToDb = new ConcurrentHashMap<>(); private transient ConcurrentHashMap fullNameToDb = new ConcurrentHashMap<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserManager.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserManager.java index c510ca99a0fcb0..b9f5b599e6b36d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserManager.java @@ -28,7 +28,7 @@ import org.apache.doris.common.PatternMatcherException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.util.QueryableReentrantReadWriteLock; +import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.mysql.MysqlPassword; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -60,7 +60,7 @@ public class UserManager implements Writable, GsonPostProcessable { public static final String ANY_HOST = "%"; private static final Logger LOG = LogManager.getLogger(UserManager.class); - private static final QueryableReentrantReadWriteLock rwLock = new QueryableReentrantReadWriteLock(false); + private static final MonitoredReentrantReadWriteLock rwLock = new MonitoredReentrantReadWriteLock(false); private static final Lock rlock = rwLock.readLock(); private static final Lock wlock = rwLock.writeLock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/QueryableReentrantLockTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/QueryableReentrantLockTest.java index f8f7b2178f971b..1608b1d6efa3e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/QueryableReentrantLockTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/QueryableReentrantLockTest.java @@ -17,6 +17,8 @@ package org.apache.doris.common.util; +import org.apache.doris.common.lock.MonitoredReentrantLock; + import org.junit.Assert; import org.junit.Test; @@ -24,7 +26,7 @@ public class QueryableReentrantLockTest { - private QueryableReentrantLock lock = new QueryableReentrantLock(true); + private MonitoredReentrantLock lock = new MonitoredReentrantLock(true); @Test public void test() throws InterruptedException { From 6e905dd9f7c68a59e375b294257d7749b39e09ff Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 8 Aug 2024 14:38:06 +0800 Subject: [PATCH 7/7] fix checkstyle --- .../apache/doris/common/lock/AbstractMonitoredLock.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java index fbe1a10e8d63a7..7389ed0d61b6b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/lock/AbstractMonitoredLock.java @@ -30,8 +30,7 @@ * a specified timeout. */ public abstract class AbstractMonitoredLock { - // Lock hold timeout in milliseconds - protected static final long HOLD_TIMEOUT = Config.max_lock_hold_threshold_seconds * 1000; + private static final Logger LOG = LoggerFactory.getLogger(AbstractMonitoredLock.class); // Thread-local variable to store the lock start time @@ -55,12 +54,12 @@ protected void afterUnlock() { if (startTime != null) { long lockHoldTimeNanos = System.nanoTime() - startTime; long lockHoldTimeMs = lockHoldTimeNanos >> 20; - if (lockHoldTimeMs > HOLD_TIMEOUT) { + if (lockHoldTimeMs > Config.max_lock_hold_threshold_seconds * 1000) { Thread currentThread = Thread.currentThread(); String stackTrace = getThreadStackTrace(currentThread.getStackTrace()); LOG.warn("Thread ID: {}, Thread Name: {} - Lock held for {} ms, exceeding hold timeout of {} ms " + "Thread stack trace:{}", - currentThread.getId(), currentThread.getName(), lockHoldTimeMs, HOLD_TIMEOUT, stackTrace); + currentThread.getId(), currentThread.getName(), lockHoldTimeMs, lockHoldTimeMs, stackTrace); } lockStartTime.remove(); }