Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow hot reloading compaction from disabled status #14470

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Allow hot reloading compaction from disabled status #14470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
63168e4
allow hot reloading compaction from disabled status
shuwenwei Dec 17, 2024
d7d9981
Modify the value setting of the parameter to conform to the comment.
shuwenwei Dec 17, 2024
6455411
fix ut
shuwenwei Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 9 additions & 14 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -770,14 +770,15 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException
"merge_interval_sec", Long.toString(conf.getMergeIntervalSec())))
.map(String::trim)
.orElse(Long.toString(conf.getMergeIntervalSec()))));
conf.setCompactionThreadCount(
int compactionThreadCount =
Integer.parseInt(
Optional.ofNullable(
properties.getProperty(
"compaction_thread_count",
Integer.toString(conf.getCompactionThreadCount())))
.map(String::trim)
.orElse(Integer.toString(conf.getCompactionThreadCount()))));
.orElse(Integer.toString(conf.getCompactionThreadCount())));
conf.setCompactionThreadCount(compactionThreadCount <= 0 ? 1 : compactionThreadCount);
int maxConcurrentAlignedSeriesInCompaction =
Integer.parseInt(
Optional.ofNullable(
Expand Down Expand Up @@ -1767,8 +1768,8 @@ private void loadCompactionHotModifiedProps(TrimProperties properties)

CompactionScheduleTaskManager.getInstance().checkAndMayApplyConfigurationChange();
// hot load compaction task manager configurations
loadCompactionIsEnabledHotModifiedProps(properties);
boolean restartCompactionTaskManager = loadCompactionThreadCountHotModifiedProps(properties);
boolean restartCompactionTaskManager = loadCompactionIsEnabledHotModifiedProps(properties);
restartCompactionTaskManager |= loadCompactionThreadCountHotModifiedProps(properties);
restartCompactionTaskManager |= loadCompactionSubTaskCountHotModifiedProps(properties);
if (restartCompactionTaskManager) {
CompactionTaskManager.getInstance().restart();
Expand Down Expand Up @@ -2071,8 +2072,7 @@ private boolean loadCompactionThreadCountHotModifiedProps(TrimProperties propert
ConfigurationFileUtils.getConfigurationDefaultValue(
"compaction_thread_count")));
if (newConfigCompactionThreadCount <= 0) {
LOGGER.error("compaction_thread_count must greater than 0");
return false;
newConfigCompactionThreadCount = 1;
}
if (newConfigCompactionThreadCount == conf.getCompactionThreadCount()) {
return false;
Expand Down Expand Up @@ -2105,8 +2105,7 @@ private boolean loadCompactionSubTaskCountHotModifiedProps(TrimProperties proper
ConfigurationFileUtils.getConfigurationDefaultValue(
"sub_compaction_thread_count")));
if (newConfigSubtaskNum <= 0) {
LOGGER.error("sub_compaction_thread_count must greater than 0");
return false;
newConfigSubtaskNum = 1;
}
if (newConfigSubtaskNum == conf.getSubCompactionTaskNum()) {
return false;
Expand All @@ -2115,7 +2114,7 @@ private boolean loadCompactionSubTaskCountHotModifiedProps(TrimProperties proper
return true;
}

private void loadCompactionIsEnabledHotModifiedProps(TrimProperties properties)
private boolean loadCompactionIsEnabledHotModifiedProps(TrimProperties properties)
throws IOException {
boolean isCompactionEnabled =
conf.isEnableSeqSpaceCompaction()
Expand Down Expand Up @@ -2159,14 +2158,10 @@ private void loadCompactionIsEnabledHotModifiedProps(TrimProperties properties)
|| newConfigEnableSeqSpaceCompaction
|| newConfigEnableUnseqSpaceCompaction;

if (!isCompactionEnabled && compactionEnabledInNewConfig) {
LOGGER.error("Compaction cannot start in current status.");
return;
}

conf.setEnableCrossSpaceCompaction(newConfigEnableCrossSpaceCompaction);
conf.setEnableSeqSpaceCompaction(newConfigEnableSeqSpaceCompaction);
conf.setEnableUnseqSpaceCompaction(newConfigEnableUnseqSpaceCompaction);
return !isCompactionEnabled && compactionEnabledInNewConfig;
}

private void loadWALHotModifiedProps(TrimProperties properties) throws IOException {
Expand Down
7 changes: 3 additions & 4 deletions ...rc/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@
import org.apache.iotdb.consensus.exception.ConsensusGroupAlreadyExistException;
import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
Expand Down Expand Up @@ -167,6 +166,7 @@
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairTaskStatus;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.settle.SettleRequestHandler;
import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate;
import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate;
Expand Down Expand Up @@ -2067,11 +2067,10 @@ public TSStatus startRepairData() throws TException {
if (!storageEngine.isReadyForNonReadWriteFunctions()) {
return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, "not all sg is ready");
}
IoTDBConfig iotdbConfig = IoTDBDescriptor.getInstance().getConfig();
if (!iotdbConfig.isEnableSeqSpaceCompaction() || !iotdbConfig.isEnableUnseqSpaceCompaction()) {
if (!CompactionTaskManager.getInstance().isInit()) {
return RpcUtils.getStatus(
TSStatusCode.EXECUTE_STATEMENT_ERROR,
"cannot start repair task because inner space compaction is not enabled");
"cannot start repair task because compaction is not enabled");
}
try {
if (storageEngine.repairData()) {
Expand Down
8 changes: 3 additions & 5 deletions ...apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,6 @@
import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.BatchProcessException;
import org.apache.iotdb.db.exception.StorageEngineException;
Expand Down Expand Up @@ -265,6 +264,7 @@
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairTaskStatus;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.trigger.service.TriggerClassLoader;
import org.apache.iotdb.pipe.api.PipePlugin;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
Expand Down Expand Up @@ -1198,12 +1198,10 @@ public SettableFuture<ConfigTaskResult> startRepairData(boolean onCluster) {
"not all sg is ready", TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()));
return future;
}
IoTDBConfig iotdbConfig = IoTDBDescriptor.getInstance().getConfig();
if (!iotdbConfig.isEnableSeqSpaceCompaction()
|| !iotdbConfig.isEnableUnseqSpaceCompaction()) {
if (!CompactionTaskManager.getInstance().isInit()) {
future.setException(
new IoTDBException(
"cannot start repair task because inner space compaction is not enabled",
"cannot start repair task because compaction is not enabled",
TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()));
return future;
}
Expand Down
11 changes: 6 additions & 5 deletions ...g/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,11 +118,7 @@ public void incrCompactionConfigVersion() {

@Override
public synchronized void start() {
if (taskExecutionPool == null
&& IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount() > 0
&& (config.isEnableSeqSpaceCompaction()
|| config.isEnableUnseqSpaceCompaction()
|| config.isEnableCrossSpaceCompaction())) {
if (!init) {
initThreadPool();
candidateCompactionTaskQueue.regsitPollLastHook(
AbstractCompactionTask::resetCompactionCandidateStatusForAllSourceFiles);
Expand All @@ -132,6 +128,10 @@ public synchronized void start() {
logger.info("Compaction task manager started.");
}

public boolean isInit() {
return this.init;
}

private void initThreadPool() {
int compactionThreadNum = IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
this.taskExecutionPool =
Expand Down Expand Up @@ -222,6 +222,7 @@ private void waitTermination() {
}
taskExecutionPool = null;
subCompactionTaskExecutionPool = null;
init = false;
storageGroupTasks.clear();
logger.info("CompactionManager stopped");
}
Expand Down
Loading