Skip to content

Commit

Permalink
Estimate data size before replication; add 3 new proto fields.
Browse files Browse the repository at this point in the history
  • Loading branch information
@smengcl
smengcl committed Aug 17, 2023
1 parent 8a6eaa8 commit 2d71626
Show file tree
Hide file tree
Showing 8 changed files with 237 additions and 27 deletions.
41 changes: 39 additions & 2 deletions hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ public class OzoneSnapshot {
private final String snapshotPath; // snapshot mask
private final String checkpointDir;
private final long referencedSize;
private final long referencedReplicatedSize;
private final long exclusiveSize;
private final long exclusiveReplicatedSize;

/**
* Constructs OzoneSnapshot from SnapshotInfo.
Expand All @@ -48,6 +51,9 @@ public class OzoneSnapshot {
* @param snapshotPath Path of the snapshot.
* @param checkpointDir Snapshot checkpoint directory.
* @param referencedSize Snapshot referenced size.
* @param referencedReplicatedSize Snapshot referenced size after replication.
* @param exclusiveSize Snapshot exclusive size.
* @param exclusiveReplicatedSize Snapshot exclusive size after replication.
*/
@SuppressWarnings("parameternumber")
public OzoneSnapshot(String volumeName,
Expand All @@ -58,7 +64,10 @@ public OzoneSnapshot(String volumeName,
UUID snapshotId,
String snapshotPath,
String checkpointDir,
long referencedSize) {
long referencedSize,
long referencedReplicatedSize,
long exclusiveSize,
long exclusiveReplicatedSize) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.name = name;
Expand All @@ -68,6 +77,9 @@ public OzoneSnapshot(String volumeName,
this.snapshotPath = snapshotPath;
this.checkpointDir = checkpointDir;
this.referencedSize = referencedSize;
this.referencedReplicatedSize = referencedReplicatedSize;
this.exclusiveSize = exclusiveSize;
this.exclusiveReplicatedSize = exclusiveReplicatedSize;
}

/**
Expand Down Expand Up @@ -149,6 +161,27 @@ public long getReferencedSize() {
return referencedSize;
}

/**
* @return Reference size after replication/EC of the snapshot
*/
public long getReferencedReplicatedSize() {
return referencedReplicatedSize;
}

/**
* @return Exclusive size of the snapshot.
*/
public long getExclusiveSize() {
return exclusiveSize;
}

/**
* @return Exclusive size after replication/EC of the snapshot.
*/
public long getExclusiveReplicatedSize() {
return exclusiveReplicatedSize;
}

public static OzoneSnapshot fromSnapshotInfo(SnapshotInfo snapshotInfo) {
return new OzoneSnapshot(
snapshotInfo.getVolumeName(),
Expand All @@ -159,6 +192,10 @@ public static OzoneSnapshot fromSnapshotInfo(SnapshotInfo snapshotInfo) {
snapshotInfo.getSnapshotId(),
snapshotInfo.getSnapshotPath(),
snapshotInfo.getCheckpointDir(),
snapshotInfo.getReferencedSize());
snapshotInfo.getReferencedSize(),
snapshotInfo.getReferencedReplicatedSize(),
snapshotInfo.getExclusiveSize(),
snapshotInfo.getExclusiveReplicatedSize()
);
}
}
39 changes: 36 additions & 3 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/QuotaUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
Expand All @@ -29,8 +31,11 @@
*/
public final class QuotaUtil {

private static final Logger LOG =
LoggerFactory.getLogger(QuotaUtil.class);

private QuotaUtil() {
};
}

/**
* From the used space and replicationConfig, calculate the expected
Expand All @@ -42,10 +47,10 @@ private QuotaUtil() {
public static long getReplicatedSize(
long dataSize, ReplicationConfig repConfig) {
if (repConfig.getReplicationType() == RATIS) {
return dataSize * ((RatisReplicationConfig)repConfig)
return dataSize * ((RatisReplicationConfig) repConfig)
.getReplicationFactor().getNumber();
} else if (repConfig.getReplicationType() == EC) {
ECReplicationConfig rc = (ECReplicationConfig)repConfig;
ECReplicationConfig rc = (ECReplicationConfig) repConfig;
int dataStripeSize = rc.getData() * rc.getEcChunkSize();
long fullStripes = dataSize / dataStripeSize;
long partialFirstChunk =
Expand All @@ -55,8 +60,36 @@ public static long getReplicatedSize(
+ partialFirstChunk * rc.getParity();
return dataSize + replicationOverhead;
} else {
LOG.warn("Unknown replication type {}. Returning original data size.",
repConfig.getReplicationType());
return dataSize;
}
}

/**
* Get an estimated data size (before replication) from the replicated size.
* An (inaccurate) reverse of getReplicatedSize().
* @param replicatedSize size after replication.
* @param repConfig The replicationConfig used to store the data.
* @return Data size before replication.
*/
public static long getDataSize(long replicatedSize,
ReplicationConfig repConfig) {
if (repConfig.getReplicationType() == RATIS) {
final int ratisReplicationFactor = ((RatisReplicationConfig) repConfig)
.getReplicationFactor().getNumber();
// May not be divisible. But it's fine to ignore remainder in our use case
return replicatedSize / ratisReplicationFactor;
} else if (repConfig.getReplicationType() == EC) {
ECReplicationConfig rc = (ECReplicationConfig) repConfig;
// Because for EC we don't know if keys have partial chunks or not,
// we assume no partial chunks so as to return an estimate.
return replicatedSize * rc.getData() / rc.getRequiredNodes();
} else {
LOG.warn("Unknown replication type {}. Returning replicatedSize.",
repConfig.getReplicationType());
return replicatedSize;
}
}

}
119 changes: 102 additions & 17 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,9 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) {
private boolean deepClean;
private boolean sstFiltered;
private long referencedSize;
private long referencedReplicatedSize;
private long exclusiveSize;
private long exclusiveReplicatedSize;

/**
* Private constructor, constructed via builder.
Expand All @@ -145,6 +148,10 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) {
* @param checkpointDir - Snapshot checkpoint directory.
* @param dbTxSequenceNumber - RDB latest transaction sequence number.
* @param deepCleaned - To be deep cleaned status for snapshot.
* @param referencedSize - Snapshot referenced size.
* @param referencedReplicatedSize - Snapshot referenced size w/ replication.
* @param exclusiveSize - Snapshot exclusive size.
* @param exclusiveReplicatedSize - Snapshot exclusive size w/ replication.
*/
@SuppressWarnings("checkstyle:ParameterNumber")
private SnapshotInfo(UUID snapshotId,
Expand All @@ -161,7 +168,10 @@ private SnapshotInfo(UUID snapshotId,
long dbTxSequenceNumber,
boolean deepCleaned,
boolean sstFiltered,
long referencedSize) {
long referencedSize,
long referencedReplicatedSize,
long exclusiveSize,
long exclusiveReplicatedSize) {
this.snapshotId = snapshotId;
this.name = name;
this.volumeName = volumeName;
Expand All @@ -177,6 +187,9 @@ private SnapshotInfo(UUID snapshotId,
this.deepClean = deepCleaned;
this.sstFiltered = sstFiltered;
this.referencedSize = referencedSize;
this.referencedReplicatedSize = referencedReplicatedSize;
this.exclusiveSize = exclusiveSize;
this.exclusiveReplicatedSize = exclusiveReplicatedSize;
}

public void setName(String name) {
Expand Down Expand Up @@ -295,7 +308,10 @@ public SnapshotInfo.Builder toBuilder() {
.setCheckpointDir(checkpointDir)
.setDeepClean(deepClean)
.setSstFiltered(sstFiltered)
.setReferencedSize(referencedSize);
.setReferencedSize(referencedSize)
.setReferencedReplicatedSize(referencedReplicatedSize)
.setExclusiveSize(exclusiveSize)
.setExclusiveReplicatedSize(exclusiveReplicatedSize);
}

/**
Expand All @@ -317,6 +333,9 @@ public static class Builder {
private boolean deepClean;
private boolean sstFiltered;
private long referencedSize;
private long referencedReplicatedSize;
private long exclusiveSize;
private long exclusiveReplicatedSize;

public Builder() {
// default values
Expand Down Expand Up @@ -398,6 +417,21 @@ public Builder setReferencedSize(long referencedSize) {
return this;
}

public Builder setReferencedReplicatedSize(long referencedReplicatedSize) {
this.referencedReplicatedSize = referencedReplicatedSize;
return this;
}

public Builder setExclusiveSize(long exclusiveSize) {
this.exclusiveSize = exclusiveSize;
return this;
}

public Builder setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
this.exclusiveReplicatedSize = exclusiveReplicatedSize;
return this;
}

public SnapshotInfo build() {
Preconditions.checkNotNull(name);
return new SnapshotInfo(
Expand All @@ -415,7 +449,10 @@ public SnapshotInfo build() {
dbTxSequenceNumber,
deepClean,
sstFiltered,
referencedSize
referencedSize,
referencedReplicatedSize,
exclusiveSize,
exclusiveReplicatedSize
);
}
}
Expand All @@ -426,15 +463,18 @@ public SnapshotInfo build() {
public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() {
OzoneManagerProtocolProtos.SnapshotInfo.Builder sib =
OzoneManagerProtocolProtos.SnapshotInfo.newBuilder()
.setSnapshotID(toProtobuf(snapshotId))
.setName(name)
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setSnapshotStatus(snapshotStatus.toProto())
.setCreationTime(creationTime)
.setDeletionTime(deletionTime)
.setSstFiltered(sstFiltered)
.setReferencedSize(referencedSize);
.setSnapshotID(toProtobuf(snapshotId))
.setName(name)
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setSnapshotStatus(snapshotStatus.toProto())
.setCreationTime(creationTime)
.setDeletionTime(deletionTime)
.setSstFiltered(sstFiltered)
.setReferencedSize(referencedSize)
.setReferencedReplicatedSize(referencedReplicatedSize)
.setExclusiveSize(exclusiveSize)
.setExclusiveReplicatedSize(exclusiveReplicatedSize);

if (pathPreviousSnapshotId != null) {
sib.setPathPreviousSnapshotID(toProtobuf(pathPreviousSnapshotId));
Expand Down Expand Up @@ -488,7 +528,23 @@ public static SnapshotInfo getFromProtobuf(
}

if (snapshotInfoProto.hasReferencedSize()) {
osib.setReferencedSize(snapshotInfoProto.getReferencedSize());
osib.setReferencedSize(
snapshotInfoProto.getReferencedSize());
}

if (snapshotInfoProto.hasReferencedReplicatedSize()) {
osib.setReferencedReplicatedSize(
snapshotInfoProto.getReferencedReplicatedSize());
}

if (snapshotInfoProto.hasExclusiveSize()) {
osib.setExclusiveSize(
snapshotInfoProto.getExclusiveSize());
}

if (snapshotInfoProto.hasExclusiveReplicatedSize()) {
osib.setExclusiveReplicatedSize(
snapshotInfoProto.getExclusiveReplicatedSize());
}

osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath())
Expand Down Expand Up @@ -551,6 +607,30 @@ public long getReferencedSize() {
return referencedSize;
}

public void setReferencedReplicatedSize(long referencedReplicatedSize) {
this.referencedReplicatedSize = referencedReplicatedSize;
}

public long getReferencedReplicatedSize() {
return referencedReplicatedSize;
}

public void setExclusiveSize(long exclusiveSize) {
this.exclusiveSize = exclusiveSize;
}

public long getExclusiveSize() {
return exclusiveSize;
}

public void setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
this.exclusiveReplicatedSize = exclusiveReplicatedSize;
}

public long getExclusiveReplicatedSize() {
return exclusiveReplicatedSize;
}

/**
* Generate default name of snapshot, (used if user doesn't provide one).
*/
Expand Down Expand Up @@ -612,9 +692,12 @@ public boolean equals(Object o) {
globalPreviousSnapshotId, that.globalPreviousSnapshotId) &&
snapshotPath.equals(that.snapshotPath) &&
checkpointDir.equals(that.checkpointDir) &&
Objects.equals(deepClean, that.deepClean) &&
Objects.equals(sstFiltered, that.sstFiltered) &&
Objects.equals(referencedSize, that.referencedSize);
deepClean == that.deepClean &&
sstFiltered == that.sstFiltered &&
referencedSize == that.referencedSize &&
referencedReplicatedSize == that.referencedReplicatedSize &&
exclusiveSize == that.exclusiveSize &&
exclusiveReplicatedSize == that.exclusiveReplicatedSize;
}

@Override
Expand All @@ -623,6 +706,8 @@ public int hashCode() {
snapshotStatus,
creationTime, deletionTime, pathPreviousSnapshotId,
globalPreviousSnapshotId, snapshotPath, checkpointDir,
deepClean, sstFiltered, referencedSize);
deepClean, sstFiltered,
referencedSize, referencedReplicatedSize,
exclusiveSize, exclusiveReplicatedSize);
}
}
Loading

0 comments on commit 2d71626

Please sign in to comment.