From 0f5a8defc65f5a8d5e61820cc6a03455c994b354 Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Fri, 12 Jan 2024 19:48:27 +0800 Subject: [PATCH] Merge master into develop branch (#5668) * feat(api):fix a concurrency issue for toString in BlockCapsule (#5657) * feat(db):optimize cache settings (#5659) * feat(lite):optimize DbLite tool (#5658) --- .../org/tron/core/store/WitnessStore.java | 22 +---- .../org/tron/consensus/ConsensusDelegate.java | 4 - .../consensus/dpos/MaintenanceManager.java | 4 +- .../main/java/org/tron/program/Version.java | 6 +- .../java/org/tron/core/db/ManagerTest.java | 17 +++- .../main/java/org/tron/plugins/DbLite.java | 93 ++++++++++++++++--- 6 files changed, 98 insertions(+), 48 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/store/WitnessStore.java b/chainbase/src/main/java/org/tron/core/store/WitnessStore.java index e01680cfc74..d23a73f92f9 100644 --- a/chainbase/src/main/java/org/tron/core/store/WitnessStore.java +++ b/chainbase/src/main/java/org/tron/core/store/WitnessStore.java @@ -11,10 +11,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import org.tron.common.cache.CacheManager; -import org.tron.common.cache.CacheStrategies; -import org.tron.common.cache.CacheType; -import org.tron.common.cache.TronCache; import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.Parameter; import org.tron.core.db.TronStoreWithRevoking; @@ -22,14 +18,10 @@ @Slf4j(topic = "DB") @Component public class WitnessStore extends TronStoreWithRevoking { - // cache for 127 SR - private final TronCache> witnessStandbyCache; @Autowired protected WitnessStore(@Value("witness") String dbName) { super(dbName); - String strategy = String.format(CacheStrategies.PATTERNS, 1, 1, "30s", 1); - witnessStandbyCache = CacheManager.allocate(CacheType.witnessStandby, strategy); } /** @@ -48,19 +40,8 @@ public WitnessCapsule get(byte[] key) { } public List getWitnessStandby() { - List list = - witnessStandbyCache.getIfPresent(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH); - if (list != null) { - return list; - } - return updateWitnessStandby(null); - } - - public List updateWitnessStandby(List all) { List ret; - if (all == null) { - all = getAllWitnesses(); - } + List all = getAllWitnesses(); all.sort(Comparator.comparingLong(WitnessCapsule::getVoteCount) .reversed().thenComparing(Comparator.comparingInt( (WitnessCapsule w) -> w.getAddress().hashCode()).reversed())); @@ -71,7 +52,6 @@ public List updateWitnessStandby(List all) { } // trim voteCount = 0 ret.removeIf(w -> w.getVoteCount() < 1); - witnessStandbyCache.put(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH, ret); return ret; } diff --git a/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java b/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java index 4a98c933bd1..767463a6a5b 100644 --- a/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java +++ b/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java @@ -108,10 +108,6 @@ public List getAllWitnesses() { return witnessStore.getAllWitnesses(); } - public List updateWitnessStandby(List all) { - return witnessStore.updateWitnessStandby(all); - } - public void saveStateFlag(int flag) { dynamicPropertiesStore.saveStateFlag(flag); } diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index fc6cdd55c15..012169bdb87 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -151,13 +151,11 @@ public void doMaintenance() { if (dynamicPropertiesStore.allowChangeDelegation()) { long nextCycle = dynamicPropertiesStore.getCurrentCycleNumber() + 1; dynamicPropertiesStore.saveCurrentCycleNumber(nextCycle); - List all = consensusDelegate.getAllWitnesses(); - all.forEach(witness -> { + consensusDelegate.getAllWitnesses().forEach(witness -> { delegationStore.setBrokerage(nextCycle, witness.createDbKey(), delegationStore.getBrokerage(witness.createDbKey())); delegationStore.setWitnessVote(nextCycle, witness.createDbKey(), witness.getVoteCount()); }); - consensusDelegate.updateWitnessStandby(all); } } diff --git a/framework/src/main/java/org/tron/program/Version.java b/framework/src/main/java/org/tron/program/Version.java index 4740daa9d5e..a01eb714297 100644 --- a/framework/src/main/java/org/tron/program/Version.java +++ b/framework/src/main/java/org/tron/program/Version.java @@ -2,9 +2,9 @@ public class Version { - public static final String VERSION_NAME = "GreatVoyage-v4.7.2-140-g9d13f9cb69"; - public static final String VERSION_CODE = "18173"; - private static final String VERSION = "4.7.3"; + public static final String VERSION_NAME = "GreatVoyage-v4.7.3-5-g788136ebe"; + public static final String VERSION_CODE = "18180"; + private static final String VERSION = "4.7.3.1"; public static String getVersion() { return VERSION; diff --git a/framework/src/test/java/org/tron/core/db/ManagerTest.java b/framework/src/test/java/org/tron/core/db/ManagerTest.java index e25faf536f7..bd79f7d9776 100755 --- a/framework/src/test/java/org/tron/core/db/ManagerTest.java +++ b/framework/src/test/java/org/tron/core/db/ManagerTest.java @@ -581,14 +581,24 @@ public void pushSwitchFork() AccountResourceInsufficientException, EventBloomException { String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62"; + String key2 = "c85ef7d79691fe79573b1a7064c19c1a9819ebdbd1faaab1a8ec92344438aaf4"; byte[] privateKey = ByteArray.fromHexString(key); final ECKey ecKey = ECKey.fromPrivate(privateKey); byte[] address = ecKey.getAddress(); - + WitnessCapsule sr1 = new WitnessCapsule( + ByteString.copyFrom(address), "www.tron.net/first"); + sr1.setVoteCount(1000000000L); + byte[] privateKey2 = ByteArray.fromHexString(key2); + final ECKey ecKey2 = ECKey.fromPrivate(privateKey2); + byte[] address2 = ecKey2.getAddress(); + WitnessCapsule sr2 = new WitnessCapsule( + ByteString.copyFrom(address2), "www.tron.net/second"); + sr2.setVoteCount(100000L); + chainManager.getWitnessStore().put(address, sr1); WitnessCapsule witnessCapsule = new WitnessCapsule(ByteString.copyFrom(address)); chainManager.getWitnessScheduleStore().saveActiveWitnesses(new ArrayList<>()); chainManager.addWitness(ByteString.copyFrom(address)); - + List witnessStandby1 = chainManager.getWitnessStore().getWitnessStandby(); Block block = getSignedBlock(witnessCapsule.getAddress(), 1533529947843L, privateKey); dbManager.pushBlock(new BlockCapsule(block)); @@ -625,6 +635,9 @@ public void pushSwitchFork() } catch (Exception e) { Assert.assertTrue(e instanceof Exception); } + chainManager.getWitnessStore().put(address, sr2); + List witnessStandby2 = chainManager.getWitnessStore().getWitnessStandby(); + Assert.assertNotEquals(witnessStandby1, witnessStandby2); } diff --git a/plugins/src/main/java/org/tron/plugins/DbLite.java b/plugins/src/main/java/org/tron/plugins/DbLite.java index 8804d6210b2..732d4913021 100644 --- a/plugins/src/main/java/org/tron/plugins/DbLite.java +++ b/plugins/src/main/java/org/tron/plugins/DbLite.java @@ -3,6 +3,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import java.io.File; import java.io.FileNotFoundException; @@ -153,10 +155,10 @@ public void generateSnapshot(String sourceDir, String snapshotDir) { long start = System.currentTimeMillis(); snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString(); try { - mergeCheckpoint(sourceDir); hasEnoughBlock(sourceDir); List snapshotDbs = getSnapshotDbs(sourceDir); split(sourceDir, snapshotDir, snapshotDbs); + mergeCheckpoint2Snapshot(sourceDir, snapshotDir); // write genesisBlock , latest recent blocks and trans fillSnapshotBlockAndTransDb(sourceDir, snapshotDir); // save min block to info @@ -190,9 +192,9 @@ public void generateHistory(String sourceDir, String historyDir) { throw new IllegalStateException( String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir)); } - mergeCheckpoint(sourceDir); hasEnoughBlock(sourceDir); split(sourceDir, historyDir, archiveDbs); + mergeCheckpoint2History(sourceDir, historyDir); // save max block to info generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(), getLatestBlockHeaderNum(sourceDir)); @@ -261,6 +263,15 @@ private List getSnapshotDbs(String sourceDir) { return snapshotDbs; } + private void mergeCheckpoint2Snapshot(String sourceDir, String historyDir) { + List snapshotDbs = getSnapshotDbs(sourceDir); + mergeCheckpoint(sourceDir, historyDir, snapshotDbs); + } + + private void mergeCheckpoint2History(String sourceDir, String destDir) { + mergeCheckpoint(sourceDir, destDir, archiveDbs); + } + private void split(String sourceDir, String destDir, List dbs) throws IOException { logger.info("Begin to split the dbs."); spec.commandLine().getOut().println("Begin to split the dbs."); @@ -278,7 +289,7 @@ private void split(String sourceDir, String destDir, List dbs) throws IO FileUtils.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs); } - private void mergeCheckpoint(String sourceDir) { + private void mergeCheckpoint(String sourceDir, String destDir, List destDbs) { logger.info("Begin to merge checkpoint to dataset."); spec.commandLine().getOut().println("Begin to merge checkpoint to dataset."); try { @@ -287,18 +298,18 @@ private void mergeCheckpoint(String sourceDir) { for (String cp : cpList) { DBInterface checkpointDb = DbTool.getDB( sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, cp); - recover(checkpointDb, sourceDir); + recover(checkpointDb, destDir, destDbs); } } else if (Paths.get(sourceDir, CHECKPOINT_DB).toFile().exists()) { DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); - recover(tmpDb, sourceDir); + recover(tmpDb, destDir, destDbs); } } catch (IOException | RocksDBException e) { throw new RuntimeException(e); } } - private void recover(DBInterface db, String destDir) + private void recover(DBInterface db, String destDir, List destDbs) throws IOException, RocksDBException { try (DBIterator iterator = db.iterator()) { for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { @@ -312,15 +323,17 @@ private void recover(DBInterface db, String destDir) byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length); byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); - DBInterface destDb = DbTool.getDB(destDir, dbName); - if (realValue != null) { - destDb.put(realKey, realValue); - } else { - byte op = value[0]; - if (DBUtils.Operator.DELETE.getValue() == op) { - destDb.delete(realKey); + if (destDbs != null && destDbs.contains(dbName)) { + DBInterface destDb = DbTool.getDB(destDir, dbName); + if (realValue != null) { + destDb.put(realKey, realValue); } else { - destDb.put(realKey, new byte[0]); + byte op = value[0]; + if (DBUtils.Operator.DELETE.getValue() == op) { + destDb.delete(realKey); + } else { + destDb.put(realKey, new byte[0]); + } } } } @@ -340,7 +353,15 @@ private void generateInfoProperties(String propertyfile, long num) } private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException { + // query latest_block_header_number from checkpoint first final String latestBlockHeaderNumber = "latest_block_header_number"; + DBInterface checkpointDb = getCheckpointDb(databaseDir); + Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb, + latestBlockHeaderNumber.getBytes()); + if (blockNumber != null) { + return blockNumber; + } + // query from propertiesDb if checkpoint not contains latest_block_header_number DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME); return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber))) .map(ByteArray::toLong) @@ -348,6 +369,14 @@ private long getLatestBlockHeaderNum(String databaseDir) throws IOException, Roc () -> new IllegalArgumentException("not found latest block header number")); } + private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) { + byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key)); + if (value != null && value.length > 1) { + return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length)); + } + return null; + } + /** * recent blocks, trans and genesis block. */ @@ -414,6 +443,15 @@ private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDB return result; } + private static byte[] simpleEncode(String s) { + byte[] bytes = s.getBytes(); + byte[] length = Ints.toByteArray(bytes.length); + byte[] r = new byte[4 + bytes.length]; + System.arraycopy(length, 0, r, 0, 4); + System.arraycopy(bytes, 0, r, 4, bytes.length); + return r; + } + private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String liteDir) throws IOException, RocksDBException { logger.info("Check the compatibility of this history."); @@ -485,6 +523,7 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo) DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME); DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME); + ProgressBar.wrap(LongStream.rangeClosed(start, end) .boxed() .sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> { @@ -519,6 +558,7 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws return; } + Path bakDir = Paths.get(liteDir, BACKUP_DIR_PREFIX + START_TIME); logger.info("Begin to merge {} to database, start {} end {}.", bakDir, start, end); spec.commandLine().getOut() @@ -545,7 +585,17 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key) throws IOException, RocksDBException { - byte[] value = DbTool.getDB(sourceDir, dbName).get(key); + DBInterface sourceDb = DbTool.getDB(sourceDir, dbName); + DBInterface checkpointDb = getCheckpointDb(sourceDir); + // get data from tmp first. + byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key)); + byte[] value; + if (isEmptyBytes(valueFromTmp)) { + value = sourceDb.get(key); + } else { + value = valueFromTmp.length == 1 + ? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length); + } if (isEmptyBytes(value)) { throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s", dbName, Arrays.toString(key))); @@ -614,6 +664,19 @@ private long getSecondBlock(String databaseDir) throws RocksDBException, IOExcep return num; } + private DBInterface getCheckpointDb(String sourceDir) throws IOException, RocksDBException { + List cpList = getCheckpointV2List(sourceDir); + DBInterface checkpointDb; + if (cpList.size() > 0) { + String latestCp = cpList.get(cpList.size() - 1); + checkpointDb = DbTool.getDB( + sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, latestCp); + } else { + checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); + } + return checkpointDb; + } + @VisibleForTesting public static void setRecentBlks(long recentBlks) { RECENT_BLKS = recentBlks;