Skip to content

Commit

Permalink
Fix typo; fix UTs.
Browse files Browse the repository at this point in the history
  • Loading branch information
smengcl committed Aug 31, 2023
1 parent c53ac0b commit 84bf6a3
Show file tree
Hide file tree
Showing 5 changed files with 56 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -729,7 +729,7 @@ public SnapshotInfo copyObject() {
.setDeepClean(deepClean)
.setSstFiltered(sstFiltered)
.setReferencedSize(referencedSize)
.setReferencedReplicatedSize(exclusiveReplicatedSize)
.setReferencedReplicatedSize(referencedReplicatedSize)
.setExclusiveSize(exclusiveSize)
.setExclusiveReplicatedSize(exclusiveReplicatedSize)
.build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -325,13 +325,26 @@ public static void addKeyToTable(boolean openKeyTable, boolean addToCache,
} else {
String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
keyName);

// Simulate bucket quota (usage) update done in OMKeyCommitRequest
String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get(
bucketKey);
omBucketInfo.incrUsedBytes(omKeyInfo.getReplicatedSize());

if (addToCache) {
omMetadataManager.getKeyTable(getDefaultBucketLayout())
.addCacheEntry(new CacheKey<>(ozoneKey),
CacheValue.get(trxnLogIndex, omKeyInfo));

omMetadataManager.getBucketTable()
.addCacheEntry(new CacheKey<>(bucketKey),
CacheValue.get(trxnLogIndex + 1, omBucketInfo));
}
omMetadataManager.getKeyTable(getDefaultBucketLayout())
.put(ozoneKey, omKeyInfo);

omMetadataManager.getBucketTable().put(bucketKey, omBucketInfo);
}
}

Expand Down Expand Up @@ -598,7 +611,6 @@ public static void addBucketToDB(String volumeName, String bucketName,
OmBucketInfo.newBuilder().setVolumeName(volumeName)
.setBucketName(bucketName)
.setBucketLayout(bucketLayout)
.setUsedBytes(3000L)
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,15 +233,18 @@ public void testValidateAndUpdateCacheQuotaBucketRecreated()
omMetadataManager);
omBucketInfo = omMetadataManager.getBucketTable().get(
bucketKey);
omBucketInfo.incrUsedBytes(1000);
final long bucketInitialUsedBytes = omBucketInfo.getUsedBytes();

omBucketInfo.incrUsedBytes(1000L);
omBucketInfo.incrUsedNamespace(100L);
omMetadataManager.getBucketTable().addCacheEntry(new CacheKey<>(bucketKey),
CacheValue.get(1L, omBucketInfo));
omMetadataManager.getBucketTable().put(bucketKey, omBucketInfo);

// prevalidate bucket
omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
Assert.assertEquals(1000L, omBucketInfo.getUsedBytes());
final long bucketExpectedUsedBytes = bucketInitialUsedBytes + 1000L;
Assert.assertEquals(bucketExpectedUsedBytes, omBucketInfo.getUsedBytes());

// perform delete
OMDirectoriesPurgeResponseWithFSO omClientResponse
Expand All @@ -252,7 +255,7 @@ public void testValidateAndUpdateCacheQuotaBucketRecreated()
// validate bucket info, no change expected
omBucketInfo = omMetadataManager.getBucketTable().get(
bucketKey);
Assert.assertEquals(1000L, omBucketInfo.getUsedBytes());
Assert.assertEquals(bucketExpectedUsedBytes, omBucketInfo.getUsedBytes());

performBatchOperationCommit(omClientResponse);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
Expand Down Expand Up @@ -187,6 +188,17 @@ public void testValidateAndUpdateCache() throws Exception {
bucketName, snapshotName1);
OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest);
String key = getTableKey(volumeName, bucketName, snapshotName1);
String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);

// Add a 1000-byte key to the bucket
OmKeyInfo key1 = addKey("key-testValidateAndUpdateCache", 12345L);
addKeyToTable(key1);

OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get(
bucketKey);
long bucketDataSize = key1.getDataSize();
long bucketUsedBytes = omBucketInfo.getUsedBytes();
assertEquals(key1.getReplicatedSize(), bucketUsedBytes);

// Value in cache should be null as of now.
assertNull(omMetadataManager.getSnapshotInfoTable().get(key));
Expand All @@ -210,12 +222,13 @@ public void testValidateAndUpdateCache() throws Exception {
.getCreateSnapshotResponse()
.getSnapshotInfo();

assertEquals(1000L, snapshotInfoProto.getReferencedSize());
assertEquals(3000L, snapshotInfoProto.getReferencedReplicatedSize());
assertEquals(bucketDataSize, snapshotInfoProto.getReferencedSize());
assertEquals(bucketUsedBytes,
snapshotInfoProto.getReferencedReplicatedSize());

SnapshotInfo snapshotInfoFromProto = getFromProtobuf(snapshotInfoProto);

// Get value form cache
// Get value from cache
SnapshotInfo snapshotInfoInCache =
omMetadataManager.getSnapshotInfoTable().get(key);
assertNotNull(snapshotInfoInCache);
Expand Down Expand Up @@ -376,12 +389,12 @@ public static OMSnapshotCreateRequest doPreExecute(

private OmKeyInfo addKey(String keyName, long objectId) {
return OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName,
HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE,
HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE,
objectId);
}

protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception {
OMRequestTestUtils.addKeyToTable(false, false, keyInfo, 0, 0L,
OMRequestTestUtils.addKeyToTable(false, true, keyInfo, 0, 0L,
omMetadataManager);
return omMetadataManager.getOzoneKey(keyInfo.getVolumeName(),
keyInfo.getBucketName(), keyInfo.getKeyName());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
import org.junit.Assert;
import org.junit.Test;

import java.io.IOException;

/**
* Test class for quota repair.
*/
Expand Down Expand Up @@ -68,6 +70,10 @@ public void testQuotaRepair() throws Exception {
fileName, omKeyInfo, -1, 50 + i, omMetadataManager);
}

// Intentionally zero out buckets' used bytes first
zeroOutBucketUsedBytes(volumeName, bucketName, 1L);
zeroOutBucketUsedBytes(volumeName, fsoBucketName, 2L);

// all count is 0 as above is adding directly to key / file table
// and directory table
OmBucketInfo obsBucketInfo = omMetadataManager.getBucketTable().get(
Expand Down Expand Up @@ -133,4 +139,16 @@ public void testQuotaRepairForOldVersionVolumeBucket() throws Exception {
Assert.assertTrue(volArgsVerify.getQuotaInBytes() == -1);
Assert.assertTrue(volArgsVerify.getQuotaInNamespace() == -1);
}

private void zeroOutBucketUsedBytes(String volumeName, String bucketName,
long trxnLogIndex)
throws IOException {
String dbKey = omMetadataManager.getBucketKey(volumeName, bucketName);
OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(dbKey);
bucketInfo.incrUsedBytes(-bucketInfo.getUsedBytes());
omMetadataManager.getBucketTable()
.addCacheEntry(new CacheKey<>(dbKey),
CacheValue.get(trxnLogIndex, bucketInfo));
omMetadataManager.getBucketTable().put(dbKey, bucketInfo);
}
}

0 comments on commit 84bf6a3

Please sign in to comment.