Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add TsfileSize column to 'show regions' #14436

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,9 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) {
.updateConfirmedConfigNodeEndPoints(
nodeId, heartbeatResp.getConfirmedConfigNodeEndPoints());
}
if (heartbeatResp.isSetRegionDisk()) {
loadManager.getLoadCache().updateRegionSizeMap(nodeId, heartbeatResp.getRegionDisk());
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ public class LoadCache {
private final Map<Integer, BaseNodeCache> nodeCacheMap;
// Map<RegionGroupId, RegionGroupCache>
private final Map<TConsensusGroupId, RegionGroupCache> regionGroupCacheMap;
// Map<NodeId, Map<RegionGroupId, RegionSize>>
private final Map<Integer, Map<Integer, Long>> regionSizeMap;
// Map<RegionGroupId, ConsensusGroupCache>
private final Map<TConsensusGroupId, ConsensusGroupCache> consensusGroupCacheMap;
// Map<DataNodeId, confirmedConfigNodes>
Expand All @@ -92,6 +94,7 @@ public LoadCache() {
this.nodeCacheMap = new ConcurrentHashMap<>();
this.heartbeatProcessingMap = new ConcurrentHashMap<>();
this.regionGroupCacheMap = new ConcurrentHashMap<>();
this.regionSizeMap = new ConcurrentHashMap<>();
this.consensusGroupCacheMap = new ConcurrentHashMap<>();
this.confirmedConfigNodeMap = new ConcurrentHashMap<>();
}
Expand Down Expand Up @@ -764,4 +767,12 @@ public void updateConfirmedConfigNodeEndPoints(
public Set<TEndPoint> getConfirmedConfigNodeEndPoints(int dataNodeId) {
return confirmedConfigNodeMap.get(dataNodeId);
}

public void updateRegionSizeMap(int dataNodeId, Map<Integer, Long> regionSizeMap) {
this.regionSizeMap.put(dataNodeId, regionSizeMap);
}

public Map<Integer, Map<Integer, Long>> getRegionSizeMap() {
return regionSizeMap;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1038,6 +1038,14 @@ public RegionInfoListResp getRegionInfoList(GetRegionInfoListPlan req) {
? RegionRoleType.Leader.toString()
: RegionRoleType.Follower.toString();
regionInfo.setRoleType(regionType);

long regionSize =
getLoadManager()
.getLoadCache()
.getRegionSizeMap()
.getOrDefault(regionInfo.getDataNodeId(), Collections.emptyMap())
.getOrDefault(regionInfo.getConsensusGroupId().getId(), -1L);
regionInfo.setTsFileSize(regionSize);
});

return regionInfoListResp;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@
import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType;
import org.apache.iotdb.db.service.DataNode;
import org.apache.iotdb.db.service.RegionMigrateService;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairTaskStatus;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager;
Expand Down Expand Up @@ -1871,6 +1872,8 @@ public TDataNodeHeartbeatResp getDataNodeHeartBeat(TDataNodeHeartbeatReq req) th
sampleDiskLoad(loadSample);

resp.setLoadSample(loadSample);

resp.setRegionDisk(FileMetrics.getInstance().getRegionSizeMap());
}
AuthorityChecker.getAuthorityFetcher().refreshToken();
resp.setHeartbeatTimestamp(req.getHeartbeatTimestamp());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.commons.schema.column.ColumnHeader;
import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo;
import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
Expand All @@ -36,10 +37,8 @@

import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BytesUtils;

import java.util.List;
Expand Down Expand Up @@ -108,9 +107,18 @@ public static void buildTSBlock(
builder
.getColumnBuilder(11)
.writeBinary(
new Binary(
DateTimeUtils.convertLongToDate(regionInfo.getCreateTime()),
TSFileConfig.STRING_CHARSET));
BytesUtils.valueOf(DateTimeUtils.convertLongToDate(regionInfo.getCreateTime())));
// region size
String regionSizeStr = "";
if (regionInfo.getConsensusGroupId().getType().ordinal()
== TConsensusGroupType.DataRegion.ordinal()) {
if (regionInfo.getTsFileSize() != -1) {
regionSizeStr = FileUtils.humanReadableByteCountSI(regionInfo.getTsFileSize());
} else {
regionSizeStr = "Unknown";
}
}
builder.getColumnBuilder(12).writeBinary(BytesUtils.valueOf(regionSizeStr));
builder.declarePosition();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.iotdb.metrics.metricsets.IMetricSet;

import java.util.List;
import java.util.Map;

@SuppressWarnings("java:S6548") // do not warn about singleton class
public class FileMetrics implements IMetricSet {
Expand Down Expand Up @@ -93,6 +94,10 @@ public void decreaseModFileSize(long size) {

// endregion

public Map<Integer, Long> getRegionSizeMap() {
return TS_FILE_METRICS.getRegionSizeMap();
}

private static class FileMetricsInstanceHolder {
private static final FileMetrics INSTANCE = new FileMetrics();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@

import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
Expand Down Expand Up @@ -433,6 +434,23 @@ private synchronized void updateRemainData(boolean seq) {

// endregion

public Map<Integer, Long> getRegionSizeMap() {
Map<Integer, Long> regionSizeMap = new HashMap<>();
for (Map<String, Pair<Long, Gauge>> map : seqFileSizeMap.values()) {
for (Map.Entry<String, Pair<Long, Gauge>> regionSizeEntry : map.entrySet()) {
Integer regionId = Integer.parseInt(regionSizeEntry.getKey());
regionSizeMap.put(regionId, regionSizeEntry.getValue().getLeft());
}
}
for (Map<String, Pair<Long, Gauge>> map : unseqFileSizeMap.values()) {
for (Map.Entry<String, Pair<Long, Gauge>> regionSizeEntry : map.entrySet()) {
Integer regionId = Integer.parseInt(regionSizeEntry.getKey());
regionSizeMap.merge(regionId, regionSizeEntry.getValue().getLeft(), Long::sum);
}
}
return regionSizeMap;
}

@TestOnly
public long getFileCount(boolean seq) {
long fileCount = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ private ColumnHeaderConstant() {
public static final String START_TIME = "StartTime";
public static final String ROLE = "Role";
public static final String CREATE_TIME = "CreateTime";
public static final String TSFILE_SIZE = "TsFileSize";

// column names for show datanodes
public static final String SCHEMA_REGION_NUM = "SchemaRegionNum";
Expand Down Expand Up @@ -353,7 +354,8 @@ private ColumnHeaderConstant() {
new ColumnHeader(RPC_PORT, TSDataType.INT32),
new ColumnHeader(INTERNAL_ADDRESS, TSDataType.TEXT),
new ColumnHeader(ROLE, TSDataType.TEXT),
new ColumnHeader(CREATE_TIME, TSDataType.TEXT));
new ColumnHeader(CREATE_TIME, TSDataType.TEXT),
new ColumnHeader(TSFILE_SIZE, TSDataType.TEXT));

public static final List<ColumnHeader> showAINodesColumnHeaders =
ImmutableList.of(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,7 @@ struct TRegionInfo {
9: optional string roleType
10: optional i64 createTime
11: optional string internalAddress
12: optional i64 tsFileSize
}

struct TShowRegionResp {
Expand Down
Loading