From 47099a229f3686d2228dbb4a29eba24e009c4245 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Thu, 14 Sep 2023 09:57:42 +0800 Subject: [PATCH 01/61] test(net): add unit test for peer connection --- .../tron/core/net/peer/PeerConnection.java | 1 - .../core/net/peer/PeerConnectionTest.java | 260 ++++++++++++++++++ .../tron/core/net/peer/PeerManagerTest.java | 138 ++++++++++ .../net/services/TronStatsManagerTest.java | 27 ++ 4 files changed, 425 insertions(+), 1 deletion(-) create mode 100644 framework/src/test/java/org/tron/core/net/peer/PeerManagerTest.java create mode 100644 framework/src/test/java/org/tron/core/net/services/TronStatsManagerTest.java diff --git a/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java b/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java index 666c99208ba..6743f00421d 100644 --- a/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java +++ b/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java @@ -204,7 +204,6 @@ public void onDisconnect() { syncBlockToFetch.clear(); syncBlockRequested.clear(); syncBlockInProcess.clear(); - syncBlockInProcess.clear(); } public String log() { diff --git a/framework/src/test/java/org/tron/core/net/peer/PeerConnectionTest.java b/framework/src/test/java/org/tron/core/net/peer/PeerConnectionTest.java index 565f2c2b817..c0d81ca2763 100644 --- a/framework/src/test/java/org/tron/core/net/peer/PeerConnectionTest.java +++ b/framework/src/test/java/org/tron/core/net/peer/PeerConnectionTest.java @@ -1,7 +1,28 @@ package org.tron.core.net.peer; +import static org.mockito.Mockito.mock; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; +import org.tron.common.overlay.message.Message; +import org.tron.common.utils.Pair; +import org.tron.common.utils.ReflectUtils; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.net.message.adv.InventoryMessage; +import org.tron.core.net.message.handshake.HelloMessage; +import org.tron.core.net.message.keepalive.PingMessage; +import org.tron.core.net.message.keepalive.PongMessage; +import org.tron.core.net.service.adv.AdvService; +import org.tron.core.net.service.sync.SyncService; +import org.tron.p2p.connection.Channel; +import org.tron.protos.Protocol; public class PeerConnectionTest { @@ -16,4 +37,243 @@ public void testVariableDefaultValue() { Assert.assertTrue(peerConnection.isNeedSyncFromUs()); Assert.assertTrue(!peerConnection.isSyncFinish()); } + + @Test + public void testOnDisconnect() { + PeerConnection peerConnection = new PeerConnection(); + + SyncService syncService = mock(SyncService.class); + ReflectUtils.setFieldValue(peerConnection, "syncService", syncService); + + AdvService advService = mock(AdvService.class); + ReflectUtils.setFieldValue(peerConnection, "advService", advService); + + Item item = new Item(Sha256Hash.ZERO_HASH, Protocol.Inventory.InventoryType.TRX); + Long time = System.currentTimeMillis(); + BlockCapsule.BlockId blockId = new BlockCapsule.BlockId(); + peerConnection.getAdvInvReceive().put(item, time); + peerConnection.getAdvInvSpread().put(item, time); + peerConnection.getSyncBlockIdCache().put(item.getHash(), time); + peerConnection.getSyncBlockToFetch().add(blockId); + peerConnection.getSyncBlockRequested().put(blockId, time); + peerConnection.getSyncBlockInProcess().add(blockId); + + peerConnection.onDisconnect(); + + //Assert.assertEquals(0, peerConnection.getAdvInvReceive().size()); + //Assert.assertEquals(0, peerConnection.getAdvInvSpread().size()); + //Assert.assertEquals(0, peerConnection.getSyncBlockIdCache().size()); + Assert.assertEquals(0, peerConnection.getSyncBlockToFetch().size()); + Assert.assertEquals(0, peerConnection.getSyncBlockRequested().size()); + Assert.assertEquals(0, peerConnection.getSyncBlockInProcess().size()); + } + + @Test + public void testIsIdle() { + PeerConnection peerConnection = new PeerConnection(); + boolean f = peerConnection.isIdle(); + Assert.assertTrue(f); + + Item item = new Item(Sha256Hash.ZERO_HASH, Protocol.Inventory.InventoryType.TRX); + Long time = System.currentTimeMillis(); + peerConnection.getAdvInvRequest().put(item, time); + f = peerConnection.isIdle(); + Assert.assertTrue(!f); + + peerConnection.getAdvInvRequest().clear(); + f = peerConnection.isIdle(); + Assert.assertTrue(f); + + BlockCapsule.BlockId blockId = new BlockCapsule.BlockId(); + peerConnection.getSyncBlockRequested().put(blockId, time); + f = peerConnection.isIdle(); + Assert.assertTrue(!f); + + peerConnection.getSyncBlockRequested().clear(); + f = peerConnection.isIdle(); + Assert.assertTrue(f); + + peerConnection.setSyncChainRequested(new Pair<>(new LinkedList<>(), time)); + f = peerConnection.isIdle(); + Assert.assertTrue(!f); + } + + @Test + public void testOnConnect() { + PeerConnection peerConnection = new PeerConnection(); + SyncService syncService = mock(SyncService.class); + ReflectUtils.setFieldValue(peerConnection, "syncService", syncService); + + HelloMessage m1 = mock(HelloMessage.class); + BlockCapsule.BlockId b1 = new BlockCapsule.BlockId(Sha256Hash.ZERO_HASH, 1); + Mockito.when(m1.getHeadBlockId()).thenReturn(b1); + + HelloMessage m2 = mock(HelloMessage.class); + BlockCapsule.BlockId b2 = new BlockCapsule.BlockId(Sha256Hash.ZERO_HASH, 2); + Mockito.when(m2.getHeadBlockId()).thenReturn(b2); + + Assert.assertTrue(peerConnection.isNeedSyncFromUs()); + Assert.assertTrue(peerConnection.isNeedSyncFromPeer()); + + ReflectUtils.setFieldValue(peerConnection, "helloMessageReceive", m1); + ReflectUtils.setFieldValue(peerConnection, "helloMessageSend", m2); + peerConnection.onConnect(); + Assert.assertTrue(peerConnection.isNeedSyncFromUs()); + Assert.assertTrue(!peerConnection.isNeedSyncFromPeer()); + + peerConnection.setNeedSyncFromPeer(true); + peerConnection.setNeedSyncFromUs(true); + ReflectUtils.setFieldValue(peerConnection, "helloMessageReceive", m2); + ReflectUtils.setFieldValue(peerConnection, "helloMessageSend", m1); + peerConnection.onConnect(); + Assert.assertTrue(!peerConnection.isNeedSyncFromUs()); + Assert.assertTrue(peerConnection.isNeedSyncFromPeer()); + + peerConnection.setNeedSyncFromPeer(true); + peerConnection.setNeedSyncFromUs(true); + ReflectUtils.setFieldValue(peerConnection, "helloMessageReceive", m1); + ReflectUtils.setFieldValue(peerConnection, "helloMessageSend", m1); + peerConnection.onConnect(); + Assert.assertTrue(!peerConnection.isNeedSyncFromUs()); + Assert.assertTrue(!peerConnection.isNeedSyncFromPeer()); + } + + @Test + public void testSetChannel() { + PeerConnection peerConnection = new PeerConnection(); + + InetSocketAddress inetSocketAddress = + new InetSocketAddress("127.0.0.2", 10001); + Channel c1 = new Channel(); + ReflectUtils.setFieldValue(c1, "inetSocketAddress", inetSocketAddress); + ReflectUtils.setFieldValue(c1, "inetAddress", inetSocketAddress.getAddress()); + + List relayNodes = new ArrayList<>(); + ReflectUtils.setFieldValue(peerConnection, "relayNodes", relayNodes); + + peerConnection.setChannel(c1); + Assert.assertTrue(!peerConnection.isRelayPeer()); + + relayNodes.add(inetSocketAddress); + peerConnection.setChannel(c1); + Assert.assertTrue(peerConnection.isRelayPeer()); + } + + @Test + public void testIsSyncFinish() { + PeerConnection peerConnection = new PeerConnection(); + boolean f = peerConnection.isSyncFinish(); + Assert.assertTrue(!f); + + peerConnection.setNeedSyncFromUs(false); + f = peerConnection.isSyncFinish(); + Assert.assertTrue(!f); + + peerConnection.setNeedSyncFromPeer(false); + f = peerConnection.isSyncFinish(); + Assert.assertTrue(f); + } + + @Test + public void testCheckAndPutAdvInvRequest() { + PeerConnection peerConnection = new PeerConnection(); + Item item = new Item(Sha256Hash.ZERO_HASH, Protocol.Inventory.InventoryType.TRX); + Long time = System.currentTimeMillis(); + boolean f = peerConnection.checkAndPutAdvInvRequest(item, time); + Assert.assertTrue(f); + + f = peerConnection.checkAndPutAdvInvRequest(item, time); + Assert.assertTrue(!f); + } + + @Test + public void testEquals() { + List relayNodes = new ArrayList<>(); + + PeerConnection p1 = new PeerConnection(); + InetSocketAddress inetSocketAddress1 = + new InetSocketAddress("127.0.0.2", 10001); + Channel c1 = new Channel(); + ReflectUtils.setFieldValue(c1, "inetSocketAddress", inetSocketAddress1); + ReflectUtils.setFieldValue(c1, "inetAddress", inetSocketAddress1.getAddress()); + ReflectUtils.setFieldValue(p1, "relayNodes", relayNodes); + p1.setChannel(c1); + + PeerConnection p2 = new PeerConnection(); + InetSocketAddress inetSocketAddress2 = + new InetSocketAddress("127.0.0.2", 10002); + Channel c2 = new Channel(); + ReflectUtils.setFieldValue(c2, "inetSocketAddress", inetSocketAddress2); + ReflectUtils.setFieldValue(c2, "inetAddress", inetSocketAddress2.getAddress()); + ReflectUtils.setFieldValue(p2, "relayNodes", relayNodes); + p2.setChannel(c2); + + PeerConnection p3 = new PeerConnection(); + InetSocketAddress inetSocketAddress3 = + new InetSocketAddress("127.0.0.2", 10002); + Channel c3 = new Channel(); + ReflectUtils.setFieldValue(c3, "inetSocketAddress", inetSocketAddress3); + ReflectUtils.setFieldValue(c3, "inetAddress", inetSocketAddress3.getAddress()); + ReflectUtils.setFieldValue(p3, "relayNodes", relayNodes); + p3.setChannel(c3); + + Assert.assertTrue(p1.equals(p1)); + Assert.assertTrue(!p1.equals(p2)); + Assert.assertTrue(p2.equals(p3)); + } + + @Test + public void testHashCode() { + List relayNodes = new ArrayList<>(); + + PeerConnection p1 = new PeerConnection(); + InetSocketAddress inetSocketAddress1 = + new InetSocketAddress("127.0.0.2", 10001); + Channel c1 = new Channel(); + ReflectUtils.setFieldValue(c1, "inetSocketAddress", inetSocketAddress1); + ReflectUtils.setFieldValue(c1, "inetAddress", inetSocketAddress1.getAddress()); + ReflectUtils.setFieldValue(p1, "relayNodes", relayNodes); + p1.setChannel(c1); + + PeerConnection p2 = new PeerConnection(); + InetSocketAddress inetSocketAddress2 = + new InetSocketAddress("127.0.0.2", 10002); + Channel c2 = new Channel(); + ReflectUtils.setFieldValue(c2, "inetSocketAddress", inetSocketAddress2); + ReflectUtils.setFieldValue(c2, "inetAddress", inetSocketAddress2.getAddress()); + ReflectUtils.setFieldValue(p2, "relayNodes", relayNodes); + p2.setChannel(c2); + + PeerConnection p3 = new PeerConnection(); + InetSocketAddress inetSocketAddress3 = + new InetSocketAddress("127.0.0.2", 10002); + Channel c3 = new Channel(); + ReflectUtils.setFieldValue(c3, "inetSocketAddress", inetSocketAddress3); + ReflectUtils.setFieldValue(c3, "inetAddress", inetSocketAddress3.getAddress()); + ReflectUtils.setFieldValue(p3, "relayNodes", relayNodes); + p3.setChannel(c3); + + Assert.assertTrue(p1.hashCode() != p2.hashCode()); + Assert.assertTrue(p2.hashCode() == p3.hashCode()); + } + + @Test + public void testNeedToLog() throws Exception { + Message msg = new PingMessage(); + boolean f = PeerConnection.needToLog(msg); + Assert.assertTrue(!f); + + msg = new PongMessage(); + f = PeerConnection.needToLog(msg); + Assert.assertTrue(!f); + + msg = new InventoryMessage(new ArrayList<>(), Protocol.Inventory.InventoryType.TRX); + f = PeerConnection.needToLog(msg); + Assert.assertTrue(!f); + + msg = new InventoryMessage(new ArrayList<>(), Protocol.Inventory.InventoryType.BLOCK); + f = PeerConnection.needToLog(msg); + Assert.assertTrue(f); + } + } diff --git a/framework/src/test/java/org/tron/core/net/peer/PeerManagerTest.java b/framework/src/test/java/org/tron/core/net/peer/PeerManagerTest.java new file mode 100644 index 00000000000..a6151da6d1c --- /dev/null +++ b/framework/src/test/java/org/tron/core/net/peer/PeerManagerTest.java @@ -0,0 +1,138 @@ +package org.tron.core.net.peer; + +import static org.mockito.Mockito.mock; + +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.context.ApplicationContext; +import org.tron.common.utils.ReflectUtils; +import org.tron.p2p.connection.Channel; + +public class PeerManagerTest { + List relayNodes = new ArrayList<>(); + + @Test + public void testAdd() throws Exception { + Field field = PeerManager.class.getDeclaredField("peers"); + field.setAccessible(true); + field.set(PeerManager.class, Collections.synchronizedList(new ArrayList<>())); + + PeerConnection p1 = new PeerConnection(); + InetSocketAddress inetSocketAddress1 = + new InetSocketAddress("127.0.0.2", 10001); + Channel c1 = new Channel(); + ReflectUtils.setFieldValue(c1, "inetSocketAddress", inetSocketAddress1); + ReflectUtils.setFieldValue(c1, "inetAddress", inetSocketAddress1.getAddress()); + ReflectUtils.setFieldValue(p1, "relayNodes", relayNodes); + p1.setChannel(c1); + + ApplicationContext ctx = mock(ApplicationContext.class); + Mockito.when(ctx.getBean(PeerConnection.class)).thenReturn(p1); + + PeerConnection p = PeerManager.add(ctx, c1); + Assert.assertTrue(p != null); + + p = PeerManager.add(ctx, c1); + Assert.assertTrue(p == null); + } + + @Test + public void testRemove() throws Exception { + Field field = PeerManager.class.getDeclaredField("peers"); + field.setAccessible(true); + field.set(PeerManager.class, Collections.synchronizedList(new ArrayList<>())); + + PeerConnection p1 = new PeerConnection(); + InetSocketAddress inetSocketAddress1 = + new InetSocketAddress("127.0.0.2", 10001); + Channel c1 = new Channel(); + ReflectUtils.setFieldValue(c1, "inetSocketAddress", inetSocketAddress1); + ReflectUtils.setFieldValue(c1, "inetAddress", inetSocketAddress1.getAddress()); + ReflectUtils.setFieldValue(p1, "relayNodes", relayNodes); + p1.setChannel(c1); + + ApplicationContext ctx = mock(ApplicationContext.class); + Mockito.when(ctx.getBean(PeerConnection.class)).thenReturn(p1); + + PeerConnection p = PeerManager.remove(c1); + Assert.assertTrue(p == null); + + PeerManager.add(ctx, c1); + p = PeerManager.remove(c1); + Assert.assertTrue(p != null); + } + + @Test + public void testGetPeerConnection() throws Exception { + Field field = PeerManager.class.getDeclaredField("peers"); + field.setAccessible(true); + field.set(PeerManager.class, Collections.synchronizedList(new ArrayList<>())); + + PeerConnection p1 = new PeerConnection(); + InetSocketAddress inetSocketAddress1 = + new InetSocketAddress("127.0.0.2", 10001); + Channel c1 = new Channel(); + ReflectUtils.setFieldValue(c1, "inetSocketAddress", inetSocketAddress1); + ReflectUtils.setFieldValue(c1, "inetAddress", inetSocketAddress1.getAddress()); + ReflectUtils.setFieldValue(p1, "relayNodes", relayNodes); + p1.setChannel(c1); + + ApplicationContext ctx = mock(ApplicationContext.class); + Mockito.when(ctx.getBean(PeerConnection.class)).thenReturn(p1); + + PeerManager.add(ctx, c1); + PeerConnection p = PeerManager.getPeerConnection(c1); + Assert.assertTrue(p != null); + } + + @Test + public void testGetPeers() throws Exception { + Field field = PeerManager.class.getDeclaredField("peers"); + field.setAccessible(true); + field.set(PeerManager.class, Collections.synchronizedList(new ArrayList<>())); + + PeerConnection p1 = new PeerConnection(); + InetSocketAddress inetSocketAddress1 = + new InetSocketAddress("127.0.0.1", 10001); + Channel c1 = new Channel(); + ReflectUtils.setFieldValue(c1, "inetSocketAddress", inetSocketAddress1); + ReflectUtils.setFieldValue(c1, "inetAddress", inetSocketAddress1.getAddress()); + ReflectUtils.setFieldValue(p1, "relayNodes", relayNodes); + p1.setChannel(c1); + + ApplicationContext ctx = mock(ApplicationContext.class); + Mockito.when(ctx.getBean(PeerConnection.class)).thenReturn(p1); + + PeerConnection p = PeerManager.add(ctx, c1); + Assert.assertTrue(p != null); + + List peers = PeerManager.getPeers(); + Assert.assertEquals(1, peers.size()); + + PeerConnection p2 = new PeerConnection(); + InetSocketAddress inetSocketAddress2 = + new InetSocketAddress("127.0.0.2", 10001); + Channel c2 = new Channel(); + ReflectUtils.setFieldValue(c2, "inetSocketAddress", inetSocketAddress2); + ReflectUtils.setFieldValue(c2, "inetAddress", inetSocketAddress2.getAddress()); + ReflectUtils.setFieldValue(p2, "relayNodes", relayNodes); + p2.setChannel(c2); + + ApplicationContext ctx2 = mock(ApplicationContext.class); + Mockito.when(ctx2.getBean(PeerConnection.class)).thenReturn(p2); + + p = PeerManager.add(ctx2, c2); + Assert.assertTrue(p != null); + + peers = PeerManager.getPeers(); + Assert.assertEquals(2, peers.size()); + } + +} diff --git a/framework/src/test/java/org/tron/core/net/services/TronStatsManagerTest.java b/framework/src/test/java/org/tron/core/net/services/TronStatsManagerTest.java new file mode 100644 index 00000000000..369955f4e7f --- /dev/null +++ b/framework/src/test/java/org/tron/core/net/services/TronStatsManagerTest.java @@ -0,0 +1,27 @@ +package org.tron.core.net.services; + +import java.net.InetAddress; +import java.net.InetSocketAddress; + +import org.junit.Assert; +import org.junit.Test; +import org.tron.core.net.service.statistics.NodeStatistics; +import org.tron.core.net.service.statistics.TronStatsManager; +import org.tron.protos.Protocol; + +public class TronStatsManagerTest { + + @Test + public void testOnDisconnect() { + InetSocketAddress inetSocketAddress = + new InetSocketAddress("127.0.0.2", 10001); + + InetAddress inetAddress = inetSocketAddress.getAddress(); + + NodeStatistics statistics = TronStatsManager.getNodeStatistics(inetAddress); + + Assert.assertTrue(null != statistics); + Assert.assertEquals(Protocol.ReasonCode.UNKNOWN, statistics.getDisconnectReason()); + } + +} From 1ec48e1666c0977b7a39d611744086c414cd2b6c Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Thu, 26 Oct 2023 15:54:38 +0800 Subject: [PATCH 02/61] feat(test): add unit tests of framework module (#5534) Co-authored-by: morgan.peng --- .../core/capsule/utils/DecodeResultTest.java | 28 +++++++ .../tron/core/capsule/utils/RLPListTest.java | 26 +++++++ .../services/jsonrpc/BlockResultTest.java | 46 ++++++++++++ .../services/jsonrpc/BuildArgumentsTest.java | 73 +++++++++++++++++++ .../services/jsonrpc/CallArgumentsTest.java | 47 ++++++++++++ .../jsonrpc/TransactionReceiptTest.java | 66 +++++++++++++++++ .../jsonrpc/TransactionResultTest.java | 59 +++++++++++++++ 7 files changed, 345 insertions(+) create mode 100644 framework/src/test/java/org/tron/core/capsule/utils/DecodeResultTest.java create mode 100644 framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java create mode 100644 framework/src/test/java/org/tron/core/services/jsonrpc/BlockResultTest.java create mode 100644 framework/src/test/java/org/tron/core/services/jsonrpc/BuildArgumentsTest.java create mode 100644 framework/src/test/java/org/tron/core/services/jsonrpc/CallArgumentsTest.java create mode 100644 framework/src/test/java/org/tron/core/services/jsonrpc/TransactionReceiptTest.java create mode 100644 framework/src/test/java/org/tron/core/services/jsonrpc/TransactionResultTest.java diff --git a/framework/src/test/java/org/tron/core/capsule/utils/DecodeResultTest.java b/framework/src/test/java/org/tron/core/capsule/utils/DecodeResultTest.java new file mode 100644 index 00000000000..008224f98a1 --- /dev/null +++ b/framework/src/test/java/org/tron/core/capsule/utils/DecodeResultTest.java @@ -0,0 +1,28 @@ +package org.tron.core.capsule.utils; + +import org.bouncycastle.util.encoders.Hex; +import org.junit.Assert; +import org.junit.Test; +import org.tron.common.utils.ByteUtil; + +public class DecodeResultTest { + + @Test + public void testConstruct() { + DecodeResult decodeResult = new DecodeResult(0, "decoded"); + Assert.assertEquals(decodeResult.getPos(), 0); + Assert.assertEquals(decodeResult.getDecoded(), "decoded"); + Assert.assertEquals(decodeResult.toString(), "decoded"); + } + + @Test + public void testToString() { + DecodeResult decodeResult = new DecodeResult(0, "decoded"); + Assert.assertEquals(decodeResult.toString(), "decoded"); + decodeResult = new DecodeResult(0, ByteUtil.intToBytes(1000)); + Assert.assertEquals(Hex.toHexString(ByteUtil.intToBytes(1000)), decodeResult.toString()); + Object[] decodedData = {"aa","bb"}; + decodeResult = new DecodeResult(0, decodedData); + Assert.assertEquals("aabb", decodeResult.toString()); + } +} diff --git a/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java b/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java new file mode 100644 index 00000000000..7d903591966 --- /dev/null +++ b/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java @@ -0,0 +1,26 @@ +package org.tron.core.capsule.utils; + +import org.junit.Assert; +import org.junit.Test; + +public class RLPListTest { + + @Test + public void testRecursivePrint() { + RLPItem element = new RLPItem("rlpItem".getBytes()); + Assert.assertEquals(new String(element.getRLPData()), "rlpItem"); + RLPList.recursivePrint(element); + RLPList rlpList = new RLPList(); + rlpList.add(new RLPItem("rlpItem0".getBytes())); + RLPList.recursivePrint(rlpList); + Assert.assertThrows(RuntimeException.class, () -> RLPList.recursivePrint(null)); + } + + @Test + public void testGetRLPData() { + RLPList rlpList = new RLPList(); + rlpList.setRLPData("rlpData".getBytes()); + Assert.assertEquals(new String(rlpList.getRLPData()), "rlpData"); + } + +} diff --git a/framework/src/test/java/org/tron/core/services/jsonrpc/BlockResultTest.java b/framework/src/test/java/org/tron/core/services/jsonrpc/BlockResultTest.java new file mode 100644 index 00000000000..2cd619a499a --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/jsonrpc/BlockResultTest.java @@ -0,0 +1,46 @@ +package org.tron.core.services.jsonrpc; + +import com.google.protobuf.ByteString; +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; +import org.tron.core.Constant; +import org.tron.core.Wallet; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.services.jsonrpc.types.BlockResult; +import org.tron.protos.Protocol; + +public class BlockResultTest extends BaseTest { + + @Resource + private Wallet wallet; + + static { + Args.setParam(new String[]{"-d", dbPath()}, Constant.TEST_CONF); + } + + @Test + public void testBlockResult() { + Protocol.Transaction.raw.Builder raw = Protocol.Transaction.raw.newBuilder(); + Protocol.Transaction.Contract.Builder contract = Protocol.Transaction.Contract.newBuilder(); + contract.setType(Protocol.Transaction.Contract.ContractType.UpdateBrokerageContract); + raw.addContract(contract.build()); + + Protocol.Transaction transaction = Protocol.Transaction.newBuilder().setRawData(raw).build(); + BlockCapsule blockCapsule = new BlockCapsule(Protocol.Block.newBuilder().setBlockHeader( + Protocol.BlockHeader.newBuilder().setRawData(Protocol.BlockHeader.raw.newBuilder() + .setParentHash(ByteString.copyFrom(ByteArray.fromHexString( + "0304f784e4e7bae517bcab94c3e0c9214fb4ac7ff9d7d5a937d1f40031f87b82"))) + .setNumber(0))).addTransactions(transaction).build()); + + BlockResult blockResult = new BlockResult(blockCapsule.getInstance(),true, wallet); + Assert.assertEquals(blockResult.getHash(), + "0x000000000000000036393ced0658419d3c251bc14ffab8d10c8b0898451054fa"); + Assert.assertEquals(blockResult.getTransactions().length, 1); + Assert.assertEquals(blockResult.getGasUsed(),"0x0"); + } + +} diff --git a/framework/src/test/java/org/tron/core/services/jsonrpc/BuildArgumentsTest.java b/framework/src/test/java/org/tron/core/services/jsonrpc/BuildArgumentsTest.java new file mode 100644 index 00000000000..f9e264c515f --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/jsonrpc/BuildArgumentsTest.java @@ -0,0 +1,73 @@ +package org.tron.core.services.jsonrpc; + +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.Wallet; +import org.tron.core.config.args.Args; +import org.tron.core.exception.JsonRpcInvalidParamsException; +import org.tron.core.exception.JsonRpcInvalidRequestException; +import org.tron.core.services.jsonrpc.types.BuildArguments; +import org.tron.core.services.jsonrpc.types.CallArguments; +import org.tron.protos.Protocol; + +public class BuildArgumentsTest extends BaseTest { + + @Resource + private Wallet wallet; + + private BuildArguments buildArguments; + + static { + Args.setParam(new String[]{"-d", dbPath()}, Constant.TEST_CONF); + } + + @Before + public void initBuildArgs() { + buildArguments = new BuildArguments( + "0x0000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000001","0x10","0.01","0x100", + "","0",9L,10000L,"",10L, + 2000L,"args",1,"",true); + } + + + @Test + public void testBuildArgument() { + CallArguments callArguments = new CallArguments( + "0x0000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000001","0x10","0.01","0x100", + "","0"); + BuildArguments buildArguments = new BuildArguments(callArguments); + Assert.assertEquals(buildArguments.getFrom(), + "0x0000000000000000000000000000000000000000"); + Assert.assertEquals(buildArguments.getTo(), + "0x0000000000000000000000000000000000000001"); + Assert.assertEquals(buildArguments.getGas(), "0x10"); + Assert.assertEquals(buildArguments.getGasPrice(), "0.01"); + } + + @Test + public void testGetContractType() + throws JsonRpcInvalidRequestException, JsonRpcInvalidParamsException { + Protocol.Transaction.Contract.ContractType contractType = + buildArguments.getContractType(wallet); + Assert.assertEquals(contractType, Protocol.Transaction.Contract.ContractType.TransferContract); + } + + @Test + public void testParseValue() throws JsonRpcInvalidParamsException { + long value = buildArguments.parseValue(); + Assert.assertEquals(value, 256L); + } + + @Test + public void testParseGas() throws JsonRpcInvalidParamsException { + long gas = buildArguments.parseGas(); + Assert.assertEquals(gas, 16L); + } + +} diff --git a/framework/src/test/java/org/tron/core/services/jsonrpc/CallArgumentsTest.java b/framework/src/test/java/org/tron/core/services/jsonrpc/CallArgumentsTest.java new file mode 100644 index 00000000000..19dd76e5e07 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/jsonrpc/CallArgumentsTest.java @@ -0,0 +1,47 @@ +package org.tron.core.services.jsonrpc; + +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.Wallet; +import org.tron.core.config.args.Args; +import org.tron.core.exception.JsonRpcInvalidParamsException; +import org.tron.core.exception.JsonRpcInvalidRequestException; +import org.tron.core.services.jsonrpc.types.CallArguments; +import org.tron.protos.Protocol; + +public class CallArgumentsTest extends BaseTest { + + @Resource + private Wallet wallet; + + private CallArguments callArguments; + + static { + Args.setParam(new String[]{"-d", dbPath()}, Constant.TEST_CONF); + } + + @Before + public void init() { + callArguments = new CallArguments("0x0000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000001","0x10","0.01","0x100", + "","0"); + } + + @Test + public void testGetContractType() + throws JsonRpcInvalidRequestException, JsonRpcInvalidParamsException { + Protocol.Transaction.Contract.ContractType contractType = callArguments.getContractType(wallet); + Assert.assertEquals(Protocol.Transaction.Contract.ContractType.TransferContract, contractType); + } + + @Test + public void testParseValue() throws JsonRpcInvalidParamsException { + long value = callArguments.parseValue(); + Assert.assertEquals(256L, value); + } + +} diff --git a/framework/src/test/java/org/tron/core/services/jsonrpc/TransactionReceiptTest.java b/framework/src/test/java/org/tron/core/services/jsonrpc/TransactionReceiptTest.java new file mode 100644 index 00000000000..f10526e30a4 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/jsonrpc/TransactionReceiptTest.java @@ -0,0 +1,66 @@ +package org.tron.core.services.jsonrpc; + +import com.google.protobuf.ByteString; +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; +import org.tron.core.Constant; +import org.tron.core.Wallet; +import org.tron.core.capsule.TransactionRetCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.services.jsonrpc.types.TransactionReceipt; +import org.tron.core.store.TransactionRetStore; +import org.tron.protos.Protocol; + +public class TransactionReceiptTest extends BaseTest { + + @Resource + private Wallet wallet; + + @Resource + private TransactionRetStore transactionRetStore; + + static { + Args.setParam(new String[]{"-d", dbPath()}, Constant.TEST_CONF); + } + + @Test + public void testTransactionReceipt() { + Protocol.TransactionInfo transactionInfo = Protocol.TransactionInfo.newBuilder() + .setId(ByteString.copyFrom("1".getBytes())) + .setContractAddress(ByteString.copyFrom("address1".getBytes())) + .setReceipt(Protocol.ResourceReceipt.newBuilder() + .setEnergyUsageTotal(0L) + .setResult(Protocol.Transaction.Result.contractResult.DEFAULT) + .build()) + .addLog(Protocol.TransactionInfo.Log.newBuilder() + .setAddress(ByteString.copyFrom("address1".getBytes())) + .setData(ByteString.copyFrom("data".getBytes())) + .build()) + .build(); + TransactionRetCapsule transactionRetCapsule = new TransactionRetCapsule(); + transactionRetCapsule.addTransactionInfo(transactionInfo); + transactionRetStore.put(ByteArray.fromLong(1), transactionRetCapsule); + + Protocol.Transaction.raw.Builder raw = Protocol.Transaction.raw.newBuilder(); + Protocol.Transaction.Contract.Builder contract = Protocol.Transaction.Contract.newBuilder(); + contract.setType(Protocol.Transaction.Contract.ContractType.UpdateBrokerageContract); + raw.addContract(contract.build()); + Protocol.Transaction transaction = Protocol.Transaction.newBuilder().setRawData(raw).build(); + + TransactionReceipt transactionReceipt = new TransactionReceipt( + Protocol.Block.newBuilder().setBlockHeader( + Protocol.BlockHeader.newBuilder().setRawData( + Protocol.BlockHeader.raw.newBuilder().setNumber(1))).addTransactions( + transaction).build(), transactionInfo, wallet); + + Assert.assertEquals(transactionReceipt.getBlockNumber(),"0x1"); + Assert.assertEquals(transactionReceipt.getTransactionIndex(),"0x0"); + Assert.assertEquals(transactionReceipt.getLogs().length,1); + Assert.assertEquals(transactionReceipt.getBlockHash(), + "0x0000000000000001464f071c8a336fd22eb5145dff1b245bda013ec89add8497"); + } + +} diff --git a/framework/src/test/java/org/tron/core/services/jsonrpc/TransactionResultTest.java b/framework/src/test/java/org/tron/core/services/jsonrpc/TransactionResultTest.java new file mode 100644 index 00000000000..a71441c73b4 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/jsonrpc/TransactionResultTest.java @@ -0,0 +1,59 @@ +package org.tron.core.services.jsonrpc; + +import com.google.protobuf.ByteString; +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; +import org.tron.core.Constant; +import org.tron.core.Wallet; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.services.jsonrpc.types.TransactionResult; +import org.tron.protos.Protocol; + +public class TransactionResultTest extends BaseTest { + + @Resource + private Wallet wallet; + + static { + Args.setParam(new String[]{"-d", dbPath()}, Constant.TEST_CONF); + } + + @Test + public void testBuildTransactionResultWithBlock() { + Protocol.Transaction.raw.Builder raw = Protocol.Transaction.raw.newBuilder().addContract( + Protocol.Transaction.Contract.newBuilder().setType( + Protocol.Transaction.Contract.ContractType.TriggerSmartContract)); + Protocol.Transaction transaction = Protocol.Transaction.newBuilder().setRawData(raw).build(); + BlockCapsule blockCapsule = new BlockCapsule(Protocol.Block.newBuilder().setBlockHeader( + Protocol.BlockHeader.newBuilder().setRawData(Protocol.BlockHeader.raw.newBuilder() + .setParentHash(ByteString.copyFrom(ByteArray.fromHexString( + "0304f784e4e7bae517bcab94c3e0c9214fb4ac7ff9d7d5a937d1f40031f87b82"))) + .setNumber(9))).addTransactions(transaction).build()); + + TransactionResult transactionResult = new TransactionResult(blockCapsule,0, transaction, + 100,1, wallet); + Assert.assertEquals(transactionResult.getBlockNumber(), "0x9"); + Assert.assertEquals(transactionResult.getHash(), + "0xdebef90d0a8077620711b1b5af2b702665887ddcbf80868108026e1ab5e0bfb7"); + Assert.assertEquals(transactionResult.getGasPrice(), "0x1"); + Assert.assertEquals(transactionResult.getGas(), "0x64"); + } + + @Test + public void testBuildTransactionResult() { + Protocol.Transaction.raw.Builder raw = Protocol.Transaction.raw.newBuilder().addContract( + Protocol.Transaction.Contract.newBuilder().setType( + Protocol.Transaction.Contract.ContractType.TriggerSmartContract)); + Protocol.Transaction transaction = Protocol.Transaction.newBuilder().setRawData(raw).build(); + TransactionResult transactionResult = new TransactionResult(transaction, wallet); + Assert.assertEquals(transactionResult.getHash(), + "0xdebef90d0a8077620711b1b5af2b702665887ddcbf80868108026e1ab5e0bfb7"); + Assert.assertEquals(transactionResult.getGasPrice(), "0x"); + Assert.assertEquals(transactionResult.getNonce(), "0x0000000000000000"); + } + +} From 8de354f8e389dfc989d247818d11d884121d625e Mon Sep 17 00:00:00 2001 From: paladim Date: Thu, 26 Oct 2023 12:35:13 +0300 Subject: [PATCH 03/61] fix(event):synchronization of sending to zeromq (#5536) Co-authored-by: paladim --- .../common/logsfilter/nativequeue/NativeMessageQueue.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java b/framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java index c0a115620c2..73dd1ee41d1 100644 --- a/framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java +++ b/framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java @@ -64,8 +64,10 @@ public void publishTrigger(String data, String topic) { } try { - publisher.sendMore(topic); - publisher.send(data); + synchronized (this) { + publisher.sendMore(topic); + publisher.send(data); + } } catch (RuntimeException e) { logger.error("write data to zeromq failed, data:{}, topic:{}, error:{}", data, topic, e.getMessage()); From df52667e48bb1aa6d0f6d6904f3eb1aea5e14e27 Mon Sep 17 00:00:00 2001 From: Kayle Date: Mon, 13 Nov 2023 14:08:53 +0800 Subject: [PATCH 04/61] feat(script): remove useless scripts (#5580) Co-authored-by: liukai --- script/checkStyle.sh | 26 ------------- script/codecov.sh | 5 --- script/querySonar.sh | 41 -------------------- script/sonar.sh | 89 -------------------------------------------- 4 files changed, 161 deletions(-) delete mode 100644 script/checkStyle.sh delete mode 100644 script/codecov.sh delete mode 100644 script/querySonar.sh delete mode 100644 script/sonar.sh diff --git a/script/checkStyle.sh b/script/checkStyle.sh deleted file mode 100644 index af3d5e7df1f..00000000000 --- a/script/checkStyle.sh +++ /dev/null @@ -1,26 +0,0 @@ -./gradlew clean build -x test > build.log 2>&1 -if [ $? != 0 ];then - echo "run ./gradlew build fail, Please check you code, Or just retry this test" - exit 1 -fi -echo "------------------------------ checkStyle check ------------------------------" -#echo $BUILDKITE_PULL_REQUEST_BASE_BRANCH -#if [[ x"$BUILDKITE_PULL_REQUEST_BASE_BRANCH" != x'develop' && x"$BUILDKITE_PULL_REQUEST_BASE_BRANCH" != x'master' ]];then -# echo "BUILDKITE_PULL_REQUEST_BASE_BRANCH isnot develop or master, SKIPED" -# exit 0 -#fi - -grep -v ":checkstyleMain\|:checkstyleTest\|:lint" build.log |grep "ant:checkstyle" > checkStyle.log 2>&1 -checkNum=`cat checkStyle.log | wc -l` -if [ ${checkNum} -gt 0 ];then - echo "please fix checkStyle problem," - echo "run [ ./gradlew clean build -x test ], and you can find checkStyle report in framework/build/reports/checkstyle/" - echo "!!!!! checkStyle Num ${checkNum} !!!!!" - cat checkStyle.log - - echo "checkStyle Failed, please fix checkStyle problem" - touch checkFailTag -else - echo "checkStyle problem zero" -fi -exit 0 diff --git a/script/codecov.sh b/script/codecov.sh deleted file mode 100644 index ba83a5a8ee2..00000000000 --- a/script/codecov.sh +++ /dev/null @@ -1,5 +0,0 @@ -bash <(curl -s https://codecov.io/bash) -t ee5ed39e-1cc4-49d1-bcf9-12bce10e5b3b -s ./common/build/reports/jacoco/test/ -s ./consensus/build/reports/jacoco/test/ -s ./chainbase/build/reports/jacoco/test/ -s ./actuator/build/reports/jacoco/test/ -s ./framework/build/reports/jacoco/test/ - -#bash <(curl -s https://codecov.io/bash) -t ee5ed39e-1cc4-49d1-bcf9-12bce10e5b3b -s ./common/build/reports/jacoco/test/jacocoTestReport.xml -s ./consensus/build/reports/jacoco/test/jacocoTestReport.xml -s ./chainbase/build/reports/jacoco/test/jacocoTestReport.xml -s ./actuator/build/reports/jacoco/test/jacocoTestReport.xml -s ./framework/build/reports/jacoco/test/jacocoTestReport.xml - -#bash <(curl -s https://codecov.io/bash) -t c2f718cbe2e84c62970a892cef614689 diff --git a/script/querySonar.sh b/script/querySonar.sh deleted file mode 100644 index 8c558634304..00000000000 --- a/script/querySonar.sh +++ /dev/null @@ -1,41 +0,0 @@ -echo "current branch is : "$BUILDKITE_BRANCH -if [ $BUILDKITE_PULL_REQUEST = "false" ]; then - SonarStatus_Url="https://sonarcloud.io/api/qualitygates/project_status?projectKey=java-tron&branch="$BUILDKITE_BRANCH - Status=`curl -s $SonarStatus_Url | jq '.projectStatus.status'` - echo "current branch sonarcloud status is : "$Status - if [ $Status = null ]; then - echo "wait for check finish, 5m ....." - sleep 300 - SonarStatus_Url="https://sonarcloud.io/api/qualitygates/project_status?projectKey=java-tron&branch="$BUILDKITE_BRANCH - Status=`curl -s $SonarStatus_Url | jq '.projectStatus.status'` - fi - - if [ x"$Status" = x'"OK"' ];then - echo "Sonar Check Pass" - exit 0 - else - echo "Sonar Check Failed" - echo "Please visit https://sonarcloud.io/dashboard?branch="$BUILDKITE_BRANCH"&id=java-tron for more details" - exit 1 - fi -else - echo "current PR number is : "$BUILDKITE_PULL_REQUEST - SonarStatus_Url="https://sonarcloud.io/api/qualitygates/project_status?projectKey=java-tron&pullRequest="$BUILDKITE_PULL_REQUEST - Status=`curl -s $SonarStatus_Url | jq '.projectStatus.status'` - if [ $Status = null ]; then - echo "wait for check finish, 5m ....." - sleep 300 - SonarStatus_Url="https://sonarcloud.io/api/qualitygates/project_status?projectKey=java-tron&pullRequest="$BUILDKITE_PULL_REQUEST - Status=`curl -s $SonarStatus_Url | jq '.projectStatus.status'` - fi - - echo "current pullRequest sonarcloud status is : "$Status - if [ x"$Status" = x'"OK"' ];then - echo "Sonar Check Pass" - exit 0 - else - echo "Sonar Check Failed" - echo "Please visit https://sonarcloud.io/dashboard?id=java-tron&pullRequest="$BUILDKITE_PULL_REQUEST" for more details" - exit 1 - fi -fi \ No newline at end of file diff --git a/script/sonar.sh b/script/sonar.sh deleted file mode 100644 index b3e2b5ccf28..00000000000 --- a/script/sonar.sh +++ /dev/null @@ -1,89 +0,0 @@ -echo "------------------------------ sonar check ------------------------------" -export SONAR_SCANNER_VERSION=4.2.0.1873 -export SONAR_SCANNER_HOME=/home/java-tron/sonar-scanner-4.1.0.1829-linux -export PATH=$SONAR_SCANNER_HOME/bin:$PATH -export SONAR_SCANNER_OPTS="-server" -#export PATH=$PATH:/home/java-tron/sonar-scanner-4.1.0.1829-linux/bin - -#BUILDKITE_BRANCH="MiraculousWang:develop" - -echo "current branch is : "$BUILDKITE_BRANCH -if [ $BUILDKITE_PULL_REQUEST = "false" ]; then - - sonar-scanner \ - -Dsonar.projectKey=java-tron \ - -Dsonar.organization=tron-zhaohong \ - -Dsonar.sources=./actuator/src,./framework/src/main,./consensus/src,./chainbase/src,./common/src,./crypto/src,./protocol/src \ - -Dsonar.java.binaries=./actuator/build/classes,./framework/build/classes,./consensus/build/classes,./chainbase/build/classes,./common/build/classes,./crypto/build/classes,./protocol/build/classes \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.links.homepage=https://github.com/tronprotocol/java-tron \ - -Dsonar.links.scm=https://github.com/tronprotocol/java-tron \ - -Dsoanr.links.issue=https://github.com/tronprotocol/java-tron/issues \ - -Dsonar.branch.name=$BUILDKITE_BRANCH \ - -Dsonar.coverage.jacoco.xmlReportPaths=./common/build/reports/jacoco/test/jacocoTestReport.xml,./consensus/build/reports/jacoco/test/jacocoTestReport.xml,./chainbase/build/reports/jacoco/test/jacocoTestReport.xml,./actuator/build/reports/jacoco/test/jacocoTestReport.xml,./framework/build/reports/jacoco/test/jacocoTestReport.xml \ - -Dsonar.login=1717c3c748ec2e0ea61e501b05458de243c4abcc > /data/checkStyle/sonar.log - - sleep 100 - - - SonarStatus_Url="https://sonarcloud.io/api/qualitygates/project_status?projectKey=java-tron&branch="$BUILDKITE_BRANCH - Status=`curl -s $SonarStatus_Url | jq '.projectStatus.status'` - echo "current branch sonarcloud status is : "$Status - if [ $Status = null ]; then - echo "wait for check finish, 5m ....." - sleep 300 - SonarStatus_Url="https://sonarcloud.io/api/qualitygates/project_status?projectKey=java-tron&branch="$BUILDKITE_BRANCH - Status=`curl -s $SonarStatus_Url | jq '.projectStatus.status'` - fi - - if [ x"$Status" = x'"OK"' ];then - echo "Sonar Check Pass" - exit 0 - else - echo ">>>>>>>>>>>>>>>>>>>>>>>>>> Sonar Check Failed <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<" - echo ">>>>>>>>>>>> Please visit https://sonarcloud.io/dashboard?branch="$BUILDKITE_BRANCH"&id=java-tron for more details <<<<<<<<<<<<<<<<<<" - touch checkFailTag - exit 0 - fi -else - echo "current PR number is : "$BUILDKITE_PULL_REQUEST - - sonar-scanner \ - -Dsonar.projectKey=java-tron \ - -Dsonar.organization=tron-zhaohong \ - -Dsonar.sources=./actuator/src,./framework/src/main,./consensus/src,./chainbase/src,./common/src,./crypto/src,./protocol/src \ - -Dsonar.java.binaries=./actuator/build/classes,./framework/build/classes,./consensus/build/classes,./chainbase/build/classes,./common/build/classes,./crypto/build/classes,./protocol/build/classes \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.links.homepage=https://github.com/tronprotocol/java-tron \ - -Dsonar.links.scm=https://github.com/tronprotocol/java-tron \ - -Dsoanr.links.issue=https://github.com/tronprotocol/java-tron/issues \ - -Dsonar.pullrequest.key=$BUILDKITE_PULL_REQUEST \ - -Dsonar.pullrequest.branch=$BUILDKITE_BRANCH \ - -Dsonar.pullrequest.base=$BUILDKITE_PULL_REQUEST_BASE_BRANCH \ - -Dsonar.coverage.jacoco.xmlReportPaths=./common/build/reports/jacoco/test/jacocoTestReport.xml,./consensus/build/reports/jacoco/test/jacocoTestReport.xml,./chainbase/build/reports/jacoco/test/jacocoTestReport.xml,./actuator/build/reports/jacoco/test/jacocoTestReport.xml,./framework/build/reports/jacoco/test/jacocoTestReport.xml \ - -Dsonar.login=1717c3c748ec2e0ea61e501b05458de243c4abcc > /data/checkStyle/sonar.log - - sleep 100 - - SonarStatus_Url="https://sonarcloud.io/api/qualitygates/project_status?projectKey=java-tron&pullRequest="$BUILDKITE_PULL_REQUEST - Status=`curl -s $SonarStatus_Url | jq '.projectStatus.status'` - if [ $Status = null ]; then - echo "wait for check finish, 5m ....." - sleep 300 - SonarStatus_Url="https://sonarcloud.io/api/qualitygates/project_status?projectKey=java-tron&pullRequest="$BUILDKITE_PULL_REQUEST - Status=`curl -s $SonarStatus_Url | jq '.projectStatus.status'` - fi - - echo "current pullRequest sonarcloud status is : "$Status - if [ x"$Status" = x'"OK"' ];then - echo "Sonar Check Pass" - exit 0 - else - echo " -------------------------------- sonar check Failed ---------------------------------" - echo ">>>>>>>>>>>>>>> Please visit https://sonarcloud.io/dashboard?id=java-tron&pullRequest="$BUILDKITE_PULL_REQUEST" for more details <<<<<<<<<<<<<<<<<<" - echo "If this Sonar problem is not caused by your modification，Make sure you local branch is newest, And merge the newest tronprotocol/java-tron" - echo ">>>>>>>>>>>>>>>>>>>>>>>>>> Sonar Check Failed <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< " - touch checkFailTag - exit 0 - fi -fi From 4557c7270f71245a4667c544edbbd582a593dec9 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Tue, 21 Nov 2023 10:52:37 +0800 Subject: [PATCH 05/61] modify default dns tree (#5585) --- framework/src/main/resources/config.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index 91f108b74a9..e13973d4fb2 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -213,7 +213,7 @@ node { dns { # dns urls to get nodes, url format tree://{pubkey}@{domain}, default empty treeUrls = [ - #"tree://APFGGTFOBVE2ZNAB3CSMNNX6RRK3ODIRLP2AA5U4YFAA6MSYZUYTQ@nodes1.example.org", + #"tree://AKMQMNAJJBL73LXWPXDI4I5ZWWIZ4AWO34DWQ636QOBBXNFXH3LQS@main.trondisco.net", //offical dns tree ] } From 2d64668d1fa254a4893966564b809a588c738770 Mon Sep 17 00:00:00 2001 From: running-tomato <31307926+tomatoishealthy@users.noreply.github.com> Date: Thu, 23 Nov 2023 17:42:33 +0800 Subject: [PATCH 06/61] feat(grpc): add reflection service (#5583) --- .../main/java/org/tron/common/parameter/CommonParameter.java | 3 +++ common/src/main/java/org/tron/core/Constant.java | 2 ++ framework/src/main/java/org/tron/core/config/args/Args.java | 4 ++++ .../src/main/java/org/tron/core/services/RpcApiService.java | 5 +++++ .../core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java | 5 +++++ .../interfaceOnSolidity/RpcApiServiceOnSolidity.java | 5 +++++ framework/src/main/resources/config-localtest.conf | 3 +++ framework/src/main/resources/config.conf | 3 +++ .../src/test/java/org/tron/common/config/args/ArgsTest.java | 1 + framework/src/test/resources/config-localtest.conf | 3 +++ framework/src/test/resources/config-test.conf | 3 +++ protocol/build.gradle | 2 ++ 12 files changed, 39 insertions(+) diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index f8f6e6c5edb..1ad50e1e359 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -255,6 +255,9 @@ public class CommonParameter { public int maxHeaderListSize; @Getter @Setter + public boolean isRpcReflectionServiceEnable; + @Getter + @Setter @Parameter(names = {"--validate-sign-thread"}, description = "Num of validate thread") public int validateSignThreadNum; @Getter diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index ba867793b41..d7ec3b715c9 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -151,6 +151,8 @@ public class Constant { public static final String NODE_RPC_MAX_HEADER_LIST_SIZE = "node.rpc.maxHeaderListSize"; + public static final String NODE_RPC_REFLECTION_SERVICE = "node.rpc.reflectionService"; + public static final String NODE_OPEN_HISTORY_QUERY_WHEN_LITEFN = "node.openHistoryQueryWhenLiteFN"; public static final String BLOCK_MAINTENANCE_TIME_INTERVAL = "block.maintenanceTimeInterval"; diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 75140dd5f54..aff6708dfff 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -748,6 +748,10 @@ public static void setParam(final String[] args, final String confFileName) { ? config.getInt(Constant.NODE_RPC_MAX_HEADER_LIST_SIZE) : GrpcUtil.DEFAULT_MAX_HEADER_LIST_SIZE; + PARAMETER.isRpcReflectionServiceEnable = + config.hasPath(Constant.NODE_RPC_REFLECTION_SERVICE) + && config.getBoolean(Constant.NODE_RPC_REFLECTION_SERVICE); + PARAMETER.maintenanceTimeInterval = config.hasPath(Constant.BLOCK_MAINTENANCE_TIME_INTERVAL) ? config .getInt(Constant.BLOCK_MAINTENANCE_TIME_INTERVAL) : 21600000L; diff --git a/framework/src/main/java/org/tron/core/services/RpcApiService.java b/framework/src/main/java/org/tron/core/services/RpcApiService.java index 94d5b97decd..2e3994c3250 100755 --- a/framework/src/main/java/org/tron/core/services/RpcApiService.java +++ b/framework/src/main/java/org/tron/core/services/RpcApiService.java @@ -8,6 +8,7 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.netty.NettyServerBuilder; +import io.grpc.protobuf.services.ProtoReflectionService; import io.grpc.stub.StreamObserver; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -250,6 +251,10 @@ public void start() { // add lite fullnode query interceptor serverBuilder.intercept(liteFnQueryGrpcInterceptor); + if (parameter.isRpcReflectionServiceEnable()) { + serverBuilder.addService(ProtoReflectionService.newInstance()); + } + apiServer = serverBuilder.build(); rateLimiterInterceptor.init(apiServer); super.start(); diff --git a/framework/src/main/java/org/tron/core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java b/framework/src/main/java/org/tron/core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java index 2f7b1dcc15c..4d801f20e5c 100755 --- a/framework/src/main/java/org/tron/core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java +++ b/framework/src/main/java/org/tron/core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java @@ -1,6 +1,7 @@ package org.tron.core.services.interfaceOnPBFT; import io.grpc.netty.NettyServerBuilder; +import io.grpc.protobuf.services.ProtoReflectionService; import io.grpc.stub.StreamObserver; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -123,6 +124,10 @@ public void start() { // add lite fullnode query interceptor serverBuilder.intercept(liteFnQueryGrpcInterceptor); + if (args.isRpcReflectionServiceEnable()) { + serverBuilder.addService(ProtoReflectionService.newInstance()); + } + apiServer = serverBuilder.build(); rateLimiterInterceptor.init(apiServer); super.start(); diff --git a/framework/src/main/java/org/tron/core/services/interfaceOnSolidity/RpcApiServiceOnSolidity.java b/framework/src/main/java/org/tron/core/services/interfaceOnSolidity/RpcApiServiceOnSolidity.java index 68ec79175fa..6bdfc824163 100755 --- a/framework/src/main/java/org/tron/core/services/interfaceOnSolidity/RpcApiServiceOnSolidity.java +++ b/framework/src/main/java/org/tron/core/services/interfaceOnSolidity/RpcApiServiceOnSolidity.java @@ -2,6 +2,7 @@ import com.google.protobuf.ByteString; import io.grpc.netty.NettyServerBuilder; +import io.grpc.protobuf.services.ProtoReflectionService; import io.grpc.stub.StreamObserver; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -124,6 +125,10 @@ public void start() { // add lite fullnode query interceptor serverBuilder.intercept(liteFnQueryGrpcInterceptor); + if (parameter.isRpcReflectionServiceEnable()) { + serverBuilder.addService(ProtoReflectionService.newInstance()); + } + apiServer = serverBuilder.build(); rateLimiterInterceptor.init(apiServer); super.start(); diff --git a/framework/src/main/resources/config-localtest.conf b/framework/src/main/resources/config-localtest.conf index 32f57481463..5ce30aeb1a4 100644 --- a/framework/src/main/resources/config-localtest.conf +++ b/framework/src/main/resources/config-localtest.conf @@ -149,6 +149,9 @@ node { # The maximum size of header list allowed to be received, default 8192 # maxHeaderListSize = + + # The switch of the reflection service, effective for all gRPC services + reflectionService = false } jsonrpc { diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index e13973d4fb2..019b6302911 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -284,6 +284,9 @@ node { # "getaccount", # "getnowblock2" # ] + + # The switch of the reflection service, effective for all gRPC services + # reflectionService = true } ## rate limiter config diff --git a/framework/src/test/java/org/tron/common/config/args/ArgsTest.java b/framework/src/test/java/org/tron/common/config/args/ArgsTest.java index b95d47bfa39..0b66064f663 100644 --- a/framework/src/test/java/org/tron/common/config/args/ArgsTest.java +++ b/framework/src/test/java/org/tron/common/config/args/ArgsTest.java @@ -48,6 +48,7 @@ public void testConfig() { RateLimiterInitialization rateLimiter = Args.getInstance().getRateLimiterInitialization(); Assert.assertEquals(rateLimiter.getHttpMap().size(), 1); Assert.assertEquals(rateLimiter.getRpcMap().size(), 0); + Assert.assertTrue(Args.getInstance().isRpcReflectionServiceEnable()); } @Test diff --git a/framework/src/test/resources/config-localtest.conf b/framework/src/test/resources/config-localtest.conf index c6ecaf28173..ff0fe8bf9d9 100644 --- a/framework/src/test/resources/config-localtest.conf +++ b/framework/src/test/resources/config-localtest.conf @@ -146,6 +146,9 @@ node { # The maximum size of header list allowed to be received, default 8192 # maxHeaderListSize = + + # The switch of the reflection service, effective for all gRPC services + reflectionService = true } jsonrpc { diff --git a/framework/src/test/resources/config-test.conf b/framework/src/test/resources/config-test.conf index cf7ca5e2f42..304ad125340 100644 --- a/framework/src/test/resources/config-test.conf +++ b/framework/src/test/resources/config-test.conf @@ -190,6 +190,9 @@ node { # The maximum size of header list allowed to be received, default 8192 # maxHeaderListSize = + + # The switch of the reflection service, effective for all gRPC services + reflectionService = true } } diff --git a/protocol/build.gradle b/protocol/build.gradle index e790e480fa0..922d6d19859 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -13,6 +13,8 @@ dependencies { compile group: 'io.grpc', name: 'grpc-netty', version: grpcVersion compile group: 'io.grpc', name: 'grpc-protobuf', version: grpcVersion compile group: 'io.grpc', name: 'grpc-stub', version: grpcVersion + compile group: 'io.grpc', name: 'grpc-services', version: grpcVersion + // end google grpc compile group: 'com.google.api.grpc', name: 'proto-google-common-protos', version: '2.15.0' From be07f5c91899cee0bfde517ac3a5e5f73c6fc957 Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Fri, 24 Nov 2023 14:36:57 +0800 Subject: [PATCH 07/61] dependency(jetty): upgrade for security (#5571) --- framework/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/build.gradle b/framework/build.gradle index c894ae1e03d..1c817f545f0 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -70,8 +70,8 @@ dependencies { compile group: 'com.google.api.grpc', name: 'proto-google-common-protos', version: '2.15.0' // http - compile 'org.eclipse.jetty:jetty-server:9.4.49.v20220914' - compile 'org.eclipse.jetty:jetty-servlet:9.4.49.v20220914' + compile 'org.eclipse.jetty:jetty-server:9.4.53.v20231009' + compile 'org.eclipse.jetty:jetty-servlet:9.4.53.v20231009' compile 'com.alibaba:fastjson:1.2.83' compile group: 'org.apache.httpcomponents', name: 'httpasyncclient', version: '4.1.1' // end http From a66316c4aa632fd9237e6ed77a972b23b46271c7 Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Wed, 29 Nov 2023 14:36:43 +0800 Subject: [PATCH 08/61] feat(test): add jetty test for CVE-2023-40167 (#5600) Co-authored-by: morgan.peng --- .../tron/common/jetty/JettyServerTest.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 framework/src/test/java/org/tron/common/jetty/JettyServerTest.java diff --git a/framework/src/test/java/org/tron/common/jetty/JettyServerTest.java b/framework/src/test/java/org/tron/common/jetty/JettyServerTest.java new file mode 100644 index 00000000000..fec29de0cb6 --- /dev/null +++ b/framework/src/test/java/org/tron/common/jetty/JettyServerTest.java @@ -0,0 +1,63 @@ +package org.tron.common.jetty; + +import java.net.URI; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.DefaultServlet; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +@Slf4j +public class JettyServerTest { + private static Server server; + private static URI serverUri; + + @BeforeClass + public static void startJetty() throws Exception { + server = new Server(); + ServerConnector connector = new ServerConnector(server); + connector.setPort(0); + server.addConnector(connector); + + ServletContextHandler context = new ServletContextHandler(); + ServletHolder defaultServ = new ServletHolder("default", DefaultServlet.class); + context.addServlet(defaultServ, "/"); + server.setHandler(context); + server.start(); + String host = connector.getHost(); + if (host == null) { + host = "localhost"; + } + int port = connector.getLocalPort(); + serverUri = new URI(String.format("http://%s:%d/", host, port)); + } + + @AfterClass + public static void stopJetty() { + try { + server.stop(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void testGet() throws Exception { + HttpClient client = new DefaultHttpClient(); + HttpGet request = new HttpGet(serverUri.resolve("/")); + request.setHeader("Content-Length", "+450"); + HttpResponse mockResponse = client.execute(request); + Assert.assertTrue(mockResponse.getStatusLine().toString().contains( + "400 Invalid Content-Length Value")); + } + +} From 3f3ffe9bc16b0e111f607e5ec3c9a0d042bb5a8e Mon Sep 17 00:00:00 2001 From: Kayle Date: Fri, 8 Dec 2023 15:31:09 +0800 Subject: [PATCH 09/61] docs(protobuf): change transaction data structure description (#5602) Co-authored-by: liukai --- Tron protobuf protocol document.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Tron protobuf protocol document.md b/Tron protobuf protocol document.md index 852ff313797..2ba2c3113a3 100644 --- a/Tron protobuf protocol document.md +++ b/Tron protobuf protocol document.md @@ -546,11 +546,11 @@ Transaction and transaction-related messages. message `raw` - `ref_block_bytes`: Deprecated. + `ref_block_bytes`: intercepted part of the now block bytes in transaction head. - `ref_block_num`: now block number in transaction head. + `ref_block_num`: Deprecated. - `ref_block_hash`: now block hash in transaction head. + `ref_block_hash`:intercepted part of the now block hash in transaction head.. `expiration`: the expiration time in transaction head. @@ -565,15 +565,15 @@ Transaction and transaction-related messages. ```java message raw { bytes ref_block_bytes = 1; - int64 ref_block_num = 3; + int64 ref_block_num = 3; bytes ref_block_hash = 4; - int64 expiration = 8; + int64 expiration = 8; repeated authority auths = 9; - bytes data = 10; + bytes data = 10; repeated Contract contract = 11; - bytes scripts = 12; + bytes scripts = 12; int64 timestamp = 14; - int64 fee_limit = 18; + int64 fee_limit = 18; } ``` From 581533769a817f6105360e0c333158a2320e455f Mon Sep 17 00:00:00 2001 From: Zoe Braiterman Date: Tue, 12 Dec 2023 22:47:56 -0500 Subject: [PATCH 10/61] fix(docs/readme): minor grammar fixes to the README file (#5606) --- README.md | 141 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 81 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 4d1d7b0b3b6..05b820297ac 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ Java implementation of the Tron Protocol -

@@ -42,6 +41,7 @@

## Table of Contents + - [What’s TRON?](#whats-tron) - [Building the Source Code](#building-the-source) - [Running java-tron](#running-java-tron) @@ -55,94 +55,109 @@ TRON is a project dedicated to building the infrastructure for a truly decentralized Internet. -* Tron Protocol, one of the largest blockchain-based operating systems in the world, offers scalable, high-availability and high-throughput support that underlies all the decentralized applications in the TRON ecosystem. +- Tron Protocol, one of the largest blockchain-based operating systems in the world, offers scalable, high-availability and high-throughput support that underlies all the decentralized applications in the TRON ecosystem. -* Tron Virtual Machine (TVM) allows anyone to develop decentralized applications (DAPPs) for themselves or their communities with smart contracts thereby making decentralized crowdfunding and token issuance easier than ever. +- Tron Virtual Machine (TVM) allows anyone to develop decentralized applications (DAPPs) for themselves or their communities with smart contracts thereby making decentralized crowdfunding and token issuance easier than ever. TRON enables large-scale development and engagement. With over 2000 transactions per second (TPS), high concurrency, low latency, and massive data transmission. It is ideal for building decentralized entertainment applications. Free features and incentive systems allow developers to create premium app experiences for users. # Building the source + Building java-tron requires `git` and 64-bit version of `Oracle JDK 1.8` to be installed, other JDK versions are not supported yet. Make sure you operate on `Linux` and `MacOS` operating systems. Clone the repo and switch to the `master` branch - ```bash - $ git clone https://github.com/tronprotocol/java-tron.git - $ cd java-tron - $ git checkout -t origin/master - ``` +```bash +$ git clone https://github.com/tronprotocol/java-tron.git +$ cd java-tron +$ git checkout -t origin/master +``` + then run the following command to build java-tron, the `FullNode.jar` file can be found in `java-tron/build/libs/` after build successful. + ```bash $ ./gradlew clean build -x test ``` - # Running java-tron + Running java-tron requires 64-bit version of `Oracle JDK 1.8` to be installed, other JDK versions are not supported yet. Make sure you operate on `Linux` and `MacOS` operating systems. -Get the mainnet configuration file: [main_net_config.conf](https://github.com/tronprotocol/tron-deployment/blob/master/main_net_config.conf), other network configuration files can be find [here](https://github.com/tronprotocol/tron-deployment). +Get the mainnet configuration file: [main_net_config.conf](https://github.com/tronprotocol/tron-deployment/blob/master/main_net_config.conf), other network configuration files can be found [here](https://github.com/tronprotocol/tron-deployment). + ## Hardware Requirements + Minimum: -* CPU with 8 cores -* 16GB RAM -* 2TB free storage space to sync the Mainnet + +- CPU with 8 cores +- 16GB RAM +- 2TB free storage space to sync the Mainnet Recommended: -* CPU with 16+ cores(32+ cores for a super representative) -* 32GB+ RAM(64GB+ for a super representative) -* High Performance SSD with at least 2.5TB free space -* 100+ MB/s download Internet service +- CPU with 16+ cores(32+ cores for a super representative) +- 32GB+ RAM(64GB+ for a super representative) +- High Performance SSD with at least 2.5TB free space +- 100+ MB/s download Internet service ## Running a full node for mainnet -Full node has full historical data, it is the entry point into the TRON network , it can be used by other processes as a gateway into the TRON network via HTTP and GRPC endpoints. You can interact with the TRON network through full node：transfer assets, deploy contracts, interact with contracts and so on. `-c ` parameter specifies a configuration file to run a full node: - ```bash - $ nohup java -Xms9G -Xmx9G -XX:ReservedCodeCacheSize=256m \ - -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ - -XX:MaxDirectMemorySize=1G -XX:+PrintGCDetails \ - -XX:+PrintGCDateStamps -Xloggc:gc.log \ - -XX:+UseConcMarkSweepGC -XX:NewRatio=2 \ - -XX:+CMSScavengeBeforeRemark -XX:+ParallelRefProcEnabled \ - -XX:+HeapDumpOnOutOfMemoryError \ - -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 \ - -jar FullNode.jar -c main_net_config.conf >> start.log 2>&1 & - ``` + +Full node has full historical data, it is the entry point into the TRON network , it can be used by other processes as a gateway into the TRON network via HTTP and GRPC endpoints. You can interact with the TRON network through full node：transfer assets, deploy contracts, interact with contracts and so on. `-c` parameter specifies a configuration file to run a full node: + +```bash +$ nohup java -Xms9G -Xmx9G -XX:ReservedCodeCacheSize=256m \ + -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ + -XX:MaxDirectMemorySize=1G -XX:+PrintGCDetails \ + -XX:+PrintGCDateStamps -Xloggc:gc.log \ + -XX:+UseConcMarkSweepGC -XX:NewRatio=2 \ + -XX:+CMSScavengeBeforeRemark -XX:+ParallelRefProcEnabled \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 \ + -jar FullNode.jar -c main_net_config.conf >> start.log 2>&1 & +``` + ## Running a super representative node for mainnet -Adding the `--witness` parameter to the startup command, full node will run as a super representative node. The super representative node supports all the functions of the full node and also supports block production. Before running, make sure you have a super representative account and get votes from others，once the number of obtained votes ranks in the top 27, your super representative node will participate in block production. -Fill in the private key of super representative address into the `localwitness` list in the `main_net_config.conf`, here is an example: - ``` - localwitness = [ - 650950B193DDDDB35B6E48912DD28F7AB0E7140C1BFDEFD493348F02295BD812 - ] - ``` +Adding the `--witness` parameter to the startup command, full node will run as a super representative node. The super representative node supports all the functions of the full node and also supports block production. Before running, make sure you have a super representative account and get votes from others. Once the number of obtained votes ranks in the top 27, your super representative node will participate in block production. + +Fill in the private key of super representative address into the `localwitness` list in the `main_net_config.conf`. Here is an example: + +``` + localwitness = [ + 650950B193DDDDB35B6E48912DD28F7AB0E7140C1BFDEFD493348F02295BD812 + ] +``` then run the following command to start the node: - ```bash - $ nohup java -Xms9G -Xmx9G -XX:ReservedCodeCacheSize=256m \ - -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ - -XX:MaxDirectMemorySize=1G -XX:+PrintGCDetails \ - -XX:+PrintGCDateStamps -Xloggc:gc.log \ - -XX:+UseConcMarkSweepGC -XX:NewRatio=2 \ - -XX:+CMSScavengeBeforeRemark -XX:+ParallelRefProcEnabled \ - -XX:+HeapDumpOnOutOfMemoryError \ - -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 \ - -jar FullNode.jar --witness -c main_net_config.conf >> start.log 2>&1 & - ``` + +```bash +$ nohup java -Xms9G -Xmx9G -XX:ReservedCodeCacheSize=256m \ + -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ + -XX:MaxDirectMemorySize=1G -XX:+PrintGCDetails \ + -XX:+PrintGCDateStamps -Xloggc:gc.log \ + -XX:+UseConcMarkSweepGC -XX:NewRatio=2 \ + -XX:+CMSScavengeBeforeRemark -XX:+ParallelRefProcEnabled \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 \ + -jar FullNode.jar --witness -c main_net_config.conf >> start.log 2>&1 & +``` ## Quick Start Tool -An easier way to build and run java-tron is to use `start.sh`, `start.sh` is a quick start script written in shell language, you can use it to build and run java-tron quickly and easily. + +An easier way to build and run java-tron is to use `start.sh`. `start.sh` is a quick start script written in the Shell language. You can use it to build and run java-tron quickly and easily. Here are some common use cases of the scripting tool -* Use `start.sh` to start a full node with the downloaded `FullNode.jar` -* Use `start.sh` to download the latest `FullNode.jar` and start a full node. -* Use `start.sh` to download the latest source code and compile a `FullNode.jar` and then start a full node. + +- Use `start.sh` to start a full node with the downloaded `FullNode.jar` +- Use `start.sh` to download the latest `FullNode.jar` and start a full node. +- Use `start.sh` to download the latest source code and compile a `FullNode.jar` and then start a full node. For more details, please refer to the tool [guide](./shell.md). ## Run inside Docker container One of the quickest ways to get `java-tron` up and running on your machine is by using Docker: + ```shell $ docker run -d --name="java-tron" \ -v /your_path/output-directory:/java-tron/output-directory \ @@ -155,6 +170,7 @@ $ docker run -d --name="java-tron" \ This will mount the `output-directory` and `logs` directories on the host, the docker.sh tool can also be used to simplify the use of docker, see more [here](docker/docker.md). # Community + [Tron Developers & SRs](https://discord.gg/hqKvyAM) is Tron's official Discord channel. Feel free to join this channel if you have any questions. [Core Devs Community](https://t.me/troncoredevscommunity) is the Telegram channel for java-tron community developers. If you want to contribute to java-tron, please join this channel. @@ -162,22 +178,27 @@ This will mount the `output-directory` and `logs` directories on the host, the d [tronprotocol/allcoredev](https://gitter.im/tronprotocol/allcoredev) is the official Gitter channel for developers. # Contribution -Thank you for considering to help out with the source code! If you'd like to contribute to java-tron, please see the [Contribution Guide](./CONTRIBUTING.md) for more details. +Thank you for considering to help out with the source code! If you'd like to contribute to java-tron, please see the [Contribution Guide](./CONTRIBUTING.md) for more details. # Resources -* [Medium](https://medium.com/@coredevs) java-tron's official technical articles are published there. -* [Documentation](https://tronprotocol.github.io/documentation-en/introduction/) java-tron's official technical documentation website. -* [Test network](http://nileex.io/) A stable test network of TRON contributed by TRON community. -* [Tronscan](https://tronscan.org/#/) TRON network blockchain browser. -* [Wallet-cli](https://github.com/tronprotocol/wallet-cli) TRON network wallet using command line. -* [TIP](https://github.com/tronprotocol/tips) TRON Improvement Proposal (TIP) describes standards for the TRON network. -* [TP](https://github.com/tronprotocol/tips/tree/master/tp) TRON Protocol (TP) describes standards already implemented in TRON network but not published as a TIP. + +- [Medium](https://medium.com/@coredevs) java-tron's official technical articles are published there. +- [Documentation](https://tronprotocol.github.io/documentation-en/introduction/) java-tron's official technical documentation website. +- [Test network](http://nileex.io/) A stable test network of TRON contributed by TRON community. +- [Tronscan](https://tronscan.org/#/) TRON network blockchain browser. +- [Wallet-cli](https://github.com/tronprotocol/wallet-cli) TRON network wallet using command line. +- [TIP](https://github.com/tronprotocol/tips) TRON Improvement Proposal (TIP) describes standards for the TRON network. +- [TP](https://github.com/tronprotocol/tips/tree/master/tp) TRON Protocol (TP) describes standards already implemented in TRON network but not published as a TIP. + # Integrity Check -* After January 3, 2023， releases are signed the gpg key: + +- After January 3, 2023， releases are signed the gpg key: ``` pub: 1254 F859 D2B1 BD9F 66E7 107D F859 BCB4 4A28 290B uid: build@tron.network ``` + # License + java-tron is released under the [LGPLv3 license](https://github.com/tronprotocol/java-tron/blob/master/LICENSE). From 30abf6cd8b2c44f2afed397e9d8a270559698009 Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Tue, 19 Dec 2023 11:30:38 +0800 Subject: [PATCH 11/61] feat(checkStyle): use https for dtd links (#5623) Co-authored-by: morgan.peng --- config/checkstyle/checkStyle.xml | 2 +- config/checkstyle/checkStyleAll.xml | 2 +- framework/config/checkstyle/checkStyle.xml | 2 +- framework/config/checkstyle/checkStyleAll.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/checkstyle/checkStyle.xml b/config/checkstyle/checkStyle.xml index 16fda0fdc4f..9d09ca11bee 100644 --- a/config/checkstyle/checkStyle.xml +++ b/config/checkstyle/checkStyle.xml @@ -1,7 +1,7 @@ + "https://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd"> - ``` [PM2] Spawning PM2 daemon with pm2_home=/root/.pm2 [PM2] PM2 Successfully daemonized @@ -166,25 +165,6 @@ If everything goes well, your terminal console output will look like following : (8) TYjQd4xrLZQGYMdLJqsTCuXVGapPqUp9ZX (10000 TRX) (9) THCw6hPZpFcLCWDcsZg3W77rXZ9rJQPncD (10000 TRX) - Private Keys - ================== - - (0) 2b2bddbeea87cecedcaf51eef55877b65725f709d2c0fcdfea0cb52d80acd52b - (1) f08759925316dc6344af538ebe3a619aeab836a0c254adca903cc764f87b0ee9 - (2) 1afc9f033cf9c6058db366b78a9f1b9c909b1b83397c9aed795afa05e9017511 - (3) f8f5bc70e91fc177eefea43b68c97b66536ac317a9300639e9d32a9db2f18a1f - (4) 031015272915917056c117d3cc2a03491a8f22ef450af83f6783efddf7064c59 - (5) 5eb25e2c1144f216aa99bbe2139d84bb6dedfb2c1ed72f3df6684a4c6d2cd96b - (6) f0b781da23992e6a3f536cb60917c3eb6a9c5434fcf441fcb8d7c58c01d6b70e - (7) 158f60a4379688a77d4a420e2f2a3e014ebf9ed0a1a093d7dc01ba23ebc5c970 - (8) e9342bb9108f46573804890a5301530c2834dce3703cd51ab77fba6161afec00 - (9) 2e9f0c507d2ea98dc4005a1afb1b743c629f7c145ccb55f38f75ae73cf8f605c - - HD Wallet - ================== - Mnemonic: border pulse twenty cruise grief shy need raw clean possible begin climb - Base HD Path: m/44'/60'/0'/0/{account_index} - ``` diff --git a/run.md b/run.md index 112478a4db5..c0ecbe4d91f 100644 --- a/run.md +++ b/run.md @@ -14,9 +14,9 @@ Use the [Testnet Config](https://github.com/tronprotocol/TronDeployment/blob/mas **Use the executable JAR(Recommended way):** ```bash -java -jar FullNode.jar -p your private key --witness -c your config.conf(Example：/data/java-tron/config.conf) +java -jar FullNode.jar -p --witness -c your config.conf(Example：/data/java-tron/config.conf) Example: -java -jar FullNode.jar -p 650950B193DDDDB35B6E48912DD28F7AB0E7140C1BFDEFD493348F02295BD812 --witness -c /data/java-tron/config.conf +java -jar FullNode.jar -p --witness -c /data/java-tron/config.conf ``` @@ -65,9 +65,9 @@ Then observe whether block synchronization success，If synchronization successf ```bash cd build/libs -java -jar FullNode.jar -p your private key --witness -c your config.conf (Example：/data/java-tron/config.conf) +java -jar FullNode.jar -p --witness -c your config.conf (Example：/data/java-tron/config.conf) Example: -java -jar FullNode.jar -p 650950B193DDDDB35B6E48912DD28F7AB0E7140C1BFDEFD493348F02295BD812 --witness -c /data/java-tron/config.conf +java -jar FullNode.jar -p --witness -c /data/java-tron/config.conf ``` @@ -81,7 +81,7 @@ java -jar FullNode.jar -p 650950B193DDDDB35B6E48912DD28F7AB0E7140C1BFDEFD493348F Using TaskInputs.file() with something that doesn't resolve to a File object has been deprecated and is scheduled to be removed in Gradle 5.0. Use TaskInputs.files() instead. > Task :run -20:39:22.749 INFO [o.t.c.c.a.Args] private.key = 63e62a71ed39e30bac7223097a173924aad5855959de517ff2987b0e0ec89f1a +20:39:22.749 INFO [o.t.c.c.a.Args] private.key = 63e62a71ed3... 20:39:22.816 WARN [o.t.c.c.a.Args] localwitness size must be one, get the first one 20:39:22.832 INFO [o.t.p.FullNode] Here is the help message.output-directory/ 三月 22, 2018 8:39:23 下午 org.tron.core.services.RpcApiService start From 20fc9b9b66730dba863ea04b84ad5a5fdccf563b Mon Sep 17 00:00:00 2001 From: running-tomato <31307926+tomatoishealthy@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:34:00 +0800 Subject: [PATCH 14/61] feat(dependency): refactor gradle dependencies (#5625) --- actuator/build.gradle | 16 ---------------- build.gradle | 13 ++++++++----- chainbase/build.gradle | 34 ++-------------------------------- common/build.gradle | 7 ------- consensus/build.gradle | 13 ------------- crypto/build.gradle | 2 -- framework/build.gradle | 23 ----------------------- plugins/build.gradle | 3 --- 8 files changed, 10 insertions(+), 101 deletions(-) diff --git a/actuator/build.gradle b/actuator/build.gradle index a0ce72b0ee6..9b200064fb0 100644 --- a/actuator/build.gradle +++ b/actuator/build.gradle @@ -1,25 +1,9 @@ description = "actuator – a series of transactions for blockchain." -// Dependency versions -// --------------------------------------- - -def junitVersion = "4.13.2" -def mockitoVersion = "2.1.0" -def testNgVersion = "6.11" -def slf4jVersion = "1.7.25" -// -------------------------------------- - dependencies { compile project(":chainbase") compile project(":protocol") compile project(":crypto") - testImplementation "junit:junit:$junitVersion" - testImplementation "org.mockito:mockito-core:$mockitoVersion" - - compile "org.slf4j:jcl-over-slf4j:$slf4jVersion" - compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - compile group: 'commons-codec', name: 'commons-codec', version: '1.11' - compile 'org.reflections:reflections:0.9.11' } test { diff --git a/build.gradle b/build.gradle index 8aad8910153..e3611a27a73 100644 --- a/build.gradle +++ b/build.gradle @@ -38,12 +38,7 @@ subprojects { dependencies { compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' compile group: 'org.slf4j', name: 'jcl-over-slf4j', version: '1.7.25' - compile "org.slf4j:jcl-over-slf4j:1.7.25" compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.9' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' compile group: 'com.google.guava', name: 'guava', version: '30.1-jre' compile "com.google.code.findbugs:jsr305:3.0.0" compile group: 'org.springframework', name: 'spring-context', version: '5.3.18' @@ -52,7 +47,15 @@ subprojects { compile group: 'org.apache.commons', name: 'commons-math', version: '2.2' compile "org.apache.commons:commons-collections4:4.1" compile group: 'joda-time', name: 'joda-time', version: '2.3' + compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' + + compileOnly 'org.projectlombok:lombok:1.18.12' + annotationProcessor 'org.projectlombok:lombok:1.18.12' + testCompileOnly 'org.projectlombok:lombok:1.18.12' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + testImplementation group: 'junit', name: 'junit', version: '4.13.2' + testImplementation "org.mockito:mockito-core:2.1.0" } task sourcesJar(type: Jar, dependsOn: classes) { diff --git a/chainbase/build.gradle b/chainbase/build.gradle index a2868aa78b4..408fe56ba42 100644 --- a/chainbase/build.gradle +++ b/chainbase/build.gradle @@ -2,46 +2,16 @@ description = "chainbase – a decentralized database for blockchain." // Dependency versions // --------------------------------------- - -def junitVersion = "4.13.2" -def mockitoVersion = "2.1.0" -def testNgVersion = "6.11" def jacocoVersion = "0.8.0" -def leveldbVersion = "1.8" def jansiVersion = "1.16" // -------------------------------------- -static def isWindows() { - return org.gradle.internal.os.OperatingSystem.current().isWindows() -} - -if (isWindows()) { - ext { - leveldbGroup = "org.ethereum" - leveldbName = "leveldbjni-all" - leveldbVersion = "1.18.3" - } -} else { - ext { - leveldbGroup = "org.fusesource.leveldbjni" - leveldbName = "leveldbjni-all" - leveldbVersion = "1.8" - } -} - dependencies { - testImplementation "junit:junit:$junitVersion" - testImplementation "org.mockito:mockito-core:$mockitoVersion" - - compile group: leveldbGroup, name: leveldbName, version: leveldbVersion - compile "org.fusesource.jansi:jansi:$jansiVersion" - compile group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' - compile group: 'com.typesafe', name: 'config', version: '1.3.2' - compile 'io.github.tronprotocol:zksnark-java-sdk:1.0.0' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.8.5' compile project(":protocol") compile project(":common") compile project(":crypto") + compile "org.fusesource.jansi:jansi:$jansiVersion" + compile 'io.github.tronprotocol:zksnark-java-sdk:1.0.0' compile 'org.reflections:reflections:0.9.11' } diff --git a/common/build.gradle b/common/build.gradle index 0431316c009..2cf15ec6efd 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -8,7 +8,6 @@ sourceCompatibility = 1.8 // Dependency versions // --------------------------------------- - def leveldbVersion = "1.8" // -------------------------------------- @@ -30,13 +29,7 @@ if (isWindows()) { } } -repositories { - mavenCentral() -} - dependencies { - testCompile group: 'junit', name: 'junit', version: '4.13.2' - compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.4.1' compile "com.cedarsoftware:java-util:1.8.0" compile group: 'org.apache.httpcomponents', name: 'httpasyncclient', version: '4.1.1' diff --git a/consensus/build.gradle b/consensus/build.gradle index 78f60bd7530..4ecd7180d13 100644 --- a/consensus/build.gradle +++ b/consensus/build.gradle @@ -1,21 +1,8 @@ description = "consensus – a distributed consensus arithmetic for blockchain." -// Dependency versions -// --------------------------------------- -def junitVersion = "4.13.2" -def mockitoVersion = "2.1.0" -def testNgVersion = "6.11" -def slf4jVersion = "1.7.25" -// -------------------------------------- - dependencies { compile project(":chainbase") compile project(":protocol") - testImplementation "junit:junit:$junitVersion" - testImplementation "org.mockito:mockito-core:$mockitoVersion" - compile "org.slf4j:jcl-over-slf4j:$slf4jVersion" - compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - compile group: 'commons-codec', name: 'commons-codec', version: '1.11' } test { diff --git a/crypto/build.gradle b/crypto/build.gradle index 5bbfc630c45..b551471bf49 100644 --- a/crypto/build.gradle +++ b/crypto/build.gradle @@ -11,8 +11,6 @@ repositories { } dependencies { - testCompile group: 'junit', name: 'junit', version: '4.13.2' - compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' compile project(":common") } diff --git a/framework/build.gradle b/framework/build.gradle index 1c817f545f0..877ed4e9073 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -40,40 +40,17 @@ dependencies { //local libraries compile fileTree(dir: 'libs', include: '*.jar') // end local libraries - testCompile group: 'junit', name: 'junit', version: '4.13.2' - testCompile group: 'org.mockito', name: 'mockito-core', version: '2.13.0' testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '1.0.0.1' testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.16.0' - compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - - compile group: 'com.typesafe', name: 'config', version: '1.3.2' - - compile "com.cedarsoftware:java-util:1.8.0" - - compile group: 'com.beust', name: 'jcommander', version: '1.72' - - compile group: 'junit', name: 'junit', version: '4.13.2' - - compile group: 'net.jcip', name: 'jcip-annotations', version: '1.0' - compile group: 'com.google.inject', name: 'guice', version: '4.1.0' - - compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.8.5' - compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.2' compile group: 'com.github.davidb', name: 'metrics-influxdb', version: '0.8.2' - compile group: 'com.carrotsearch', name: 'java-sizeof', version: '0.0.5' - - //compile 'com.googlecode.cqengine:cqengine:2.12.4' - compile group: 'com.google.api.grpc', name: 'proto-google-common-protos', version: '2.15.0' - // http compile 'org.eclipse.jetty:jetty-server:9.4.53.v20231009' compile 'org.eclipse.jetty:jetty-servlet:9.4.53.v20231009' compile 'com.alibaba:fastjson:1.2.83' - compile group: 'org.apache.httpcomponents', name: 'httpasyncclient', version: '4.1.1' // end http // https://mvnrepository.com/artifact/com.github.briandilley.jsonrpc4j/jsonrpc4j diff --git a/plugins/build.gradle b/plugins/build.gradle index 3085bb24bdc..7f226d7099c 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -26,9 +26,6 @@ configurations.getByName('checkstyleConfig') { dependencies { //local libraries compile fileTree(dir: 'libs', include: '*.jar') - testCompile group: 'junit', name: 'junit', version: '4.13.2' - testCompile group: 'org.mockito', name: 'mockito-core', version: '2.13.0' - testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '1.0.0.1' testCompile project(":framework") testCompile project(":framework").sourceSets.test.output compile group: 'info.picocli', name: 'picocli', version: '4.6.3' From acfaac819a3fdcad50611cdcae26d44a626ea473 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Wed, 27 Dec 2023 11:55:14 +0800 Subject: [PATCH 15/61] feat(net):fix a concurrency issue for toString in BlockCapsule (#5594) --- .../org/tron/core/capsule/BlockCapsule.java | 3 +-- .../tron/core/capsule/BlockCapsuleTest.java | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/capsule/BlockCapsule.java b/chainbase/src/main/java/org/tron/core/capsule/BlockCapsule.java index 8baafcb5dd5..01ff7fb5365 100755 --- a/chainbase/src/main/java/org/tron/core/capsule/BlockCapsule.java +++ b/chainbase/src/main/java/org/tron/core/capsule/BlockCapsule.java @@ -56,7 +56,6 @@ public class BlockCapsule implements ProtoCapsule { private Block block; private List transactions = new ArrayList<>(); - private StringBuilder toStringBuff = new StringBuilder(); private boolean isSwitch; @Getter @Setter @@ -314,7 +313,7 @@ public boolean hasWitnessSignature() { @Override public String toString() { - toStringBuff.setLength(0); + StringBuilder toStringBuff = new StringBuilder(); toStringBuff.append("BlockCapsule \n[ "); toStringBuff.append("hash=").append(getBlockId()).append("\n"); diff --git a/framework/src/test/java/org/tron/core/capsule/BlockCapsuleTest.java b/framework/src/test/java/org/tron/core/capsule/BlockCapsuleTest.java index 0dd078dedb0..3c86d893895 100644 --- a/framework/src/test/java/org/tron/core/capsule/BlockCapsuleTest.java +++ b/framework/src/test/java/org/tron/core/capsule/BlockCapsuleTest.java @@ -2,7 +2,9 @@ import com.google.protobuf.ByteString; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.junit.AfterClass; import org.junit.Assert; @@ -146,4 +148,20 @@ public void testGetTimeStamp() { Assert.assertEquals(1234L, blockCapsule0.getTimeStamp()); } + @Test + public void testConcurrentToString() throws InterruptedException { + List threadList = new ArrayList<>(); + int n = 10; + for (int i = 0; i < n; i++) { + threadList.add(new Thread(() -> blockCapsule0.toString())); + } + for (int i = 0; i < n; i++) { + threadList.get(i).start(); + } + for (int i = 0; i < n; i++) { + threadList.get(i).join(); + } + Assert.assertTrue(true); + } + } From 565aef5d58121a3a25d12d999301c592be53440f Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Wed, 27 Dec 2023 12:01:47 +0800 Subject: [PATCH 16/61] feat(api): print more details for unexpected error (#5601) --- framework/src/main/java/org/tron/core/services/http/Util.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/services/http/Util.java b/framework/src/main/java/org/tron/core/services/http/Util.java index 8f27521495a..2b6b929d8a0 100644 --- a/framework/src/main/java/org/tron/core/services/http/Util.java +++ b/framework/src/main/java/org/tron/core/services/http/Util.java @@ -475,7 +475,7 @@ public static String getMemo(byte[] memo) { } public static void processError(Exception e, HttpServletResponse response) { - logger.debug("Exception: {}", e.getMessage()); + logger.debug(e.getMessage(), e); try { response.getWriter().println(Util.printErrorMsg(e)); } catch (IOException ioe) { From a76e17704e99dd8cbbb4d38979d07d2d7c0306ac Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Wed, 27 Dec 2023 12:11:49 +0800 Subject: [PATCH 17/61] test(plugins/dbLite): add test for merge function (#5603) --- plugins/src/test/java/org/tron/plugins/DbLiteTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java index 8baf48b0b48..5cb8c26c4bc 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java @@ -117,8 +117,8 @@ void testTools(String dbType, int checkpointVersion) cli.execute(argsForSnapshot); // start fullNode startApp(); - // produce transactions for 6 seconds - generateSomeTransactions(6); + // produce transactions + generateSomeTransactions(checkpointVersion == 1 ? 6 : 18); // stop the node shutdown(); // generate history @@ -139,7 +139,7 @@ void testTools(String dbType, int checkpointVersion) } // start and validate the snapshot startApp(); - generateSomeTransactions(6); + generateSomeTransactions(checkpointVersion == 1 ? 18 : 6); // stop the node shutdown(); // merge history From c4d7635200d3cac0f59cfd0c29b7a1324e65b3d2 Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Wed, 27 Dec 2023 12:15:26 +0800 Subject: [PATCH 18/61] feat(log/api): separate unexpected and expected logs (#5624) --- .../java/org/tron/core/services/RpcApiService.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/core/services/RpcApiService.java b/framework/src/main/java/org/tron/core/services/RpcApiService.java index 2e3994c3250..c5077facf6f 100755 --- a/framework/src/main/java/org/tron/core/services/RpcApiService.java +++ b/framework/src/main/java/org/tron/core/services/RpcApiService.java @@ -2475,9 +2475,13 @@ public void createShieldedContractParameters( ShieldedTRC20Parameters shieldedTRC20Parameters = wallet .createShieldedContractParameters(request); responseObserver.onNext(shieldedTRC20Parameters); + } catch (ZksnarkException | ContractValidateException e) { + responseObserver.onError(getRunTimeException(e)); + logger.info("createShieldedContractParameters: {}", e.getMessage()); + return; } catch (Exception e) { responseObserver.onError(getRunTimeException(e)); - logger.info("createShieldedContractParameters exception caught: " + e.getMessage()); + logger.error("createShieldedContractParameters: ", e); return; } responseObserver.onCompleted(); @@ -2518,9 +2522,13 @@ public void scanShieldedTRC20NotesByIvk( request.getNk().toByteArray(), request.getEventsList()); responseObserver.onNext(decryptNotes); + } catch (BadItemException | ZksnarkException e) { + responseObserver.onError(getRunTimeException(e)); + logger.info("scanShieldedTRC20NotesByIvk: {}", e.getMessage()); + return; } catch (Exception e) { responseObserver.onError(getRunTimeException(e)); - logger.info("scanShieldedTRC20NotesByIvk exception caught: " + e.getMessage()); + logger.error("scanShieldedTRC20NotesByIvk:", e); return; } responseObserver.onCompleted(); From 989115a03f3d1c72a31486bba3f0e0846b3bbb6f Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Wed, 27 Dec 2023 12:22:04 +0800 Subject: [PATCH 19/61] feat(net): add codeVersion for HelloMessage (#5584) --- .../core/net/message/handshake/HelloMessage.java | 12 ++++++++++-- .../tron/core/net/services/HandShakeServiceTest.java | 4 ++++ protocol/src/main/protos/core/Tron.proto | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java b/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java index 9efb223fdb7..b98d0e173fb 100755 --- a/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java @@ -11,6 +11,7 @@ import org.tron.core.net.message.MessageTypes; import org.tron.core.net.message.TronMessage; import org.tron.p2p.discover.Node; +import org.tron.program.Version; import org.tron.protos.Discover.Endpoint; import org.tron.protos.Protocol; import org.tron.protos.Protocol.HelloMessage.Builder; @@ -61,6 +62,7 @@ public HelloMessage(Node from, long timestamp, ChainBaseManager chainBaseManager builder.setNodeType(chainBaseManager.getNodeType().getType()); builder.setLowestBlockNum(chainBaseManager.isLiteNode() ? chainBaseManager.getLowestBlockNum() : 0); + builder.setCodeVersion(ByteString.copyFrom(Version.getVersion().getBytes())); this.helloMessage = builder.build(); this.type = MessageTypes.P2P_HELLO.asByte(); @@ -127,17 +129,23 @@ public String toString() { .append("lowestBlockNum: ").append(helloMessage.getLowestBlockNum()).append("\n"); ByteString address = helloMessage.getAddress(); - if (address != null && !address.isEmpty()) { + if (!address.isEmpty()) { builder.append("address:") .append(StringUtil.encode58Check(address.toByteArray())).append("\n"); } ByteString signature = helloMessage.getSignature(); - if (signature != null && !signature.isEmpty()) { + if (!signature.isEmpty()) { builder.append("signature:") .append(signature.toByteArray().length).append("\n"); } + ByteString codeVersion = helloMessage.getCodeVersion(); + if (!codeVersion.isEmpty()) { + builder.append("codeVersion:") + .append(new String(codeVersion.toByteArray())).append("\n"); + } + return builder.toString(); } diff --git a/framework/src/test/java/org/tron/core/net/services/HandShakeServiceTest.java b/framework/src/test/java/org/tron/core/net/services/HandShakeServiceTest.java index e027749458f..ea2156ae713 100644 --- a/framework/src/test/java/org/tron/core/net/services/HandShakeServiceTest.java +++ b/framework/src/test/java/org/tron/core/net/services/HandShakeServiceTest.java @@ -38,6 +38,7 @@ import org.tron.p2p.connection.Channel; import org.tron.p2p.discover.Node; import org.tron.p2p.utils.NetUtil; +import org.tron.program.Version; import org.tron.protos.Discover.Endpoint; import org.tron.protos.Protocol; import org.tron.protos.Protocol.HelloMessage.Builder; @@ -99,6 +100,9 @@ public void testOkHelloMessage() Node node = new Node(NetUtil.getNodeId(), a1.getAddress().getHostAddress(), null, a1.getPort()); HelloMessage helloMessage = new HelloMessage(node, System.currentTimeMillis(), ChainBaseManager.getChainBaseManager()); + + Assert.assertEquals(Version.getVersion(), + new String(helloMessage.getHelloMessage().getCodeVersion().toByteArray())); method.invoke(p2pEventHandler, peer, helloMessage.getSendBytes()); //dup hello message diff --git a/protocol/src/main/protos/core/Tron.proto b/protocol/src/main/protos/core/Tron.proto index 2fc08901e93..41ef968d907 100644 --- a/protocol/src/main/protos/core/Tron.proto +++ b/protocol/src/main/protos/core/Tron.proto @@ -626,6 +626,7 @@ message HelloMessage { bytes signature = 8; int32 nodeType = 9; int64 lowestBlockNum = 10; + bytes codeVersion = 11; } message InternalTransaction { From 0d48db4870390f2afdb19bd4ca51a9baf7e4a334 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Thu, 28 Dec 2023 10:41:49 +0800 Subject: [PATCH 20/61] =?UTF-8?q?feat(net):=C2=A0stop=20broadcasting=20tra?= =?UTF-8?q?nsactions=20when=20the=20block=20cannot=20be=20solidified?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/parameter/CommonParameter.java | 8 +++ .../src/main/java/org/tron/core/Constant.java | 4 ++ .../src/main/java/org/tron/core/Wallet.java | 7 +++ .../java/org/tron/core/config/args/Args.java | 10 ++++ .../org/tron/core/net/TronNetDelegate.java | 15 ++++++ .../messagehandler/InventoryMsgHandler.java | 7 +++ .../org/tron/core/config/args/ArgsTest.java | 2 + .../tron/core/net/TronNetDelegateTest.java | 51 +++++++++++++++++++ .../InventoryMsgHandlerTest.java | 22 +++++++- protocol/src/main/protos/api/api.proto | 2 + 10 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 framework/src/test/java/org/tron/core/net/TronNetDelegateTest.java diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index f8f6e6c5edb..319dd025c90 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -651,6 +651,14 @@ public class CommonParameter { @Setter public long allowCancelAllUnfreezeV2; + @Getter + @Setter + public boolean unsolidifiedBlockCheck; + + @Getter + @Setter + public int maxUnsolidifiedBlocks; + private static double calcMaxTimeRatio() { //return max(2.0, min(5.0, 5 * 4.0 / max(Runtime.getRuntime().availableProcessors(), 1))); return 5.0; diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index ba867793b41..96e99b72079 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -371,4 +371,8 @@ public class Constant { public static final String DYNAMIC_CONFIG_CHECK_INTERVAL = "node.dynamicConfig.checkInterval"; public static final String COMMITTEE_ALLOW_TVM_SHANGHAI = "committee.allowTvmShangHai"; + + public static final String UNSOLIDIFIED_BLOCK_CHECK = "node.unsolidifiedBlockCheck"; + + public static final String MAX_UNSOLIDIFIED_BLOCKS = "node.maxUnsolidifiedBlocks"; } diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index 0865e202974..a36ea5855ec 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -520,6 +520,13 @@ public GrpcAPI.Return broadcastTransaction(Transaction signedTransaction) { } } + if (tronNetDelegate.unsolidifiedBlockCheck()) { + logger.warn("Broadcast transaction {} has failed, block unsolidified.", txID); + return builder.setResult(false).setCode(response_code.BLOCK_UNSOLIDIFIED) + .setMessage(ByteString.copyFromUtf8("Bock unsolidified.")) + .build(); + } + if (dbManager.isTooManyPending()) { logger.warn("Broadcast transaction {} has failed, too many pending.", txID); return builder.setResult(false).setCode(response_code.SERVER_BUSY) diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 75140dd5f54..0ca6b004fa7 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -230,6 +230,8 @@ public static void clearParam() { PARAMETER.dynamicConfigEnable = false; PARAMETER.dynamicConfigCheckInterval = 600; PARAMETER.allowTvmShangHai = 0; + PARAMETER.unsolidifiedBlockCheck = true; + PARAMETER.maxUnsolidifiedBlocks = 1000; } /** @@ -1178,6 +1180,14 @@ public static void setParam(final String[] args, final String confFileName) { config.hasPath(Constant.COMMITTEE_ALLOW_TVM_SHANGHAI) ? config .getInt(Constant.COMMITTEE_ALLOW_TVM_SHANGHAI) : 0; + PARAMETER.unsolidifiedBlockCheck = + !config.hasPath(Constant.UNSOLIDIFIED_BLOCK_CHECK) + || config.getBoolean(Constant.UNSOLIDIFIED_BLOCK_CHECK); + + PARAMETER.maxUnsolidifiedBlocks = + config.hasPath(Constant.MAX_UNSOLIDIFIED_BLOCKS) ? config + .getInt(Constant.MAX_UNSOLIDIFIED_BLOCKS) : 1000; + logConfig(); } diff --git a/framework/src/main/java/org/tron/core/net/TronNetDelegate.java b/framework/src/main/java/org/tron/core/net/TronNetDelegate.java index 5d09e67908d..3aa06362cc9 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetDelegate.java +++ b/framework/src/main/java/org/tron/core/net/TronNetDelegate.java @@ -29,6 +29,7 @@ import org.tron.core.capsule.BlockCapsule.BlockId; import org.tron.core.capsule.PbftSignCapsule; import org.tron.core.capsule.TransactionCapsule; +import org.tron.core.config.args.Args; import org.tron.core.db.Manager; import org.tron.core.exception.AccountResourceInsufficientException; import org.tron.core.exception.BadBlockException; @@ -98,6 +99,11 @@ public class TronNetDelegate { @Setter private volatile boolean exit = true; + private int maxUnsolidifiedBlocks = Args.getInstance().getMaxUnsolidifiedBlocks(); + + private boolean unsolidifiedBlockCheck + = Args.getInstance().isUnsolidifiedBlockCheck(); + private Cache freshBlockId = CacheBuilder.newBuilder() .maximumSize(blockIdCacheSize).expireAfterWrite(1, TimeUnit.HOURS) .recordStats().build(); @@ -365,4 +371,13 @@ public long getMaintenanceTimeInterval() { return chainBaseManager.getDynamicPropertiesStore().getMaintenanceTimeInterval(); } + public boolean unsolidifiedBlockCheck() { + if (!unsolidifiedBlockCheck) { + return false; + } + long headNum = chainBaseManager.getHeadBlockNum(); + long solidNum = chainBaseManager.getSolidBlockId().getNum(); + return headNum - solidNum >= maxUnsolidifiedBlocks; + } + } diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java index 02b04d73b32..40b0db33482 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java @@ -43,6 +43,7 @@ public void processMessage(PeerConnection peer, TronMessage msg) { } private boolean check(PeerConnection peer, InventoryMessage inventoryMessage) { + InventoryType type = inventoryMessage.getInventoryType(); int size = inventoryMessage.getHashList().size(); @@ -52,6 +53,12 @@ private boolean check(PeerConnection peer, InventoryMessage inventoryMessage) { return false; } + if (type.equals(InventoryType.TRX) && tronNetDelegate.unsolidifiedBlockCheck()) { + logger.warn("Drop inv: {} size: {} from Peer {}, block unsolidified", + type, size, peer.getInetAddress()); + return false; + } + if (type.equals(InventoryType.TRX) && transactionsMsgHandler.isBusy()) { logger.warn("Drop inv: {} size: {} from Peer {}, transactionsMsgHandler is busy", type, size, peer.getInetAddress()); diff --git a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java index 13e272d1e13..78d3ea0c6e6 100644 --- a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java +++ b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java @@ -102,6 +102,8 @@ public void get() { Assert.assertEquals(0, parameter.getActiveNodes().size()); Assert.assertEquals(30, parameter.getMaxConnections()); Assert.assertEquals(43, parameter.getNodeP2pVersion()); + Assert.assertEquals(1000, parameter.getMaxUnsolidifiedBlocks()); + Assert.assertEquals(true, parameter.isUnsolidifiedBlockCheck()); //Assert.assertEquals(30, args.getSyncNodeCount()); // gRPC network configs checking diff --git a/framework/src/test/java/org/tron/core/net/TronNetDelegateTest.java b/framework/src/test/java/org/tron/core/net/TronNetDelegateTest.java new file mode 100644 index 00000000000..674c13a73fe --- /dev/null +++ b/framework/src/test/java/org/tron/core/net/TronNetDelegateTest.java @@ -0,0 +1,51 @@ +package org.tron.core.net; + +import static org.mockito.Mockito.mock; + +import java.lang.reflect.Field; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.tron.common.parameter.CommonParameter; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.ChainBaseManager; +import org.tron.core.Constant; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.config.args.Args; + +public class TronNetDelegateTest { + + @Test + public void test() throws Exception { + Args.setParam(new String[] {"-w"}, Constant.TEST_CONF); + CommonParameter parameter = Args.getInstance(); + Args.logConfig(); + + BlockCapsule.BlockId blockId = new BlockCapsule.BlockId(Sha256Hash.ZERO_HASH, 10000L); + + TronNetDelegate tronNetDelegate = new TronNetDelegate(); + + ChainBaseManager chainBaseManager = mock(ChainBaseManager.class); + Mockito.when(chainBaseManager.getHeadBlockNum()).thenReturn(10000L); + Mockito.when(chainBaseManager.getSolidBlockId()).thenReturn(blockId); + + Field field = tronNetDelegate.getClass().getDeclaredField("chainBaseManager"); + field.setAccessible(true); + field.set(tronNetDelegate, chainBaseManager); + + Assert.assertTrue(!tronNetDelegate.unsolidifiedBlockCheck()); + + blockId = new BlockCapsule.BlockId(Sha256Hash.ZERO_HASH, 1L); + Mockito.when(chainBaseManager.getSolidBlockId()).thenReturn(blockId); + Assert.assertTrue(tronNetDelegate.unsolidifiedBlockCheck()); + + parameter.setUnsolidifiedBlockCheck(false); + tronNetDelegate = new TronNetDelegate(); + + field = tronNetDelegate.getClass().getDeclaredField("unsolidifiedBlockCheck"); + field.setAccessible(true); + field.set(tronNetDelegate, false); + + Assert.assertTrue(!tronNetDelegate.unsolidifiedBlockCheck()); + } +} diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/InventoryMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/InventoryMsgHandlerTest.java index 97db6207b2a..ee16d333193 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/InventoryMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/InventoryMsgHandlerTest.java @@ -1,10 +1,16 @@ package org.tron.core.net.messagehandler; +import static org.mockito.Mockito.mock; + import java.lang.reflect.Field; import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; import org.junit.Test; +import org.mockito.Mockito; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; +import org.tron.core.net.TronNetDelegate; import org.tron.core.net.message.adv.InventoryMessage; import org.tron.core.net.peer.PeerConnection; import org.tron.p2p.connection.Channel; @@ -12,10 +18,12 @@ public class InventoryMsgHandlerTest { - private InventoryMsgHandler handler = new InventoryMsgHandler(); - @Test public void testProcessMessage() throws Exception { + InventoryMsgHandler handler = new InventoryMsgHandler(); + Args.setParam(new String[] {"-w"}, Constant.TEST_CONF); + Args.logConfig(); + InventoryMessage msg = new InventoryMessage(new ArrayList<>(), InventoryType.TRX); PeerConnection peer = new PeerConnection(); peer.setChannel(getChannel("1.0.0.3", 1000)); @@ -31,6 +39,16 @@ public void testProcessMessage() throws Exception { peer.setNeedSyncFromUs(true); handler.processMessage(peer, msg); + peer.setNeedSyncFromUs(false); + + TronNetDelegate tronNetDelegate = mock(TronNetDelegate.class); + Mockito.when(tronNetDelegate.unsolidifiedBlockCheck()).thenReturn(true); + + Field field = handler.getClass().getDeclaredField("tronNetDelegate"); + field.setAccessible(true); + field.set(handler, tronNetDelegate); + + handler.processMessage(peer, msg); } private Channel getChannel(String host, int port) throws Exception { diff --git a/protocol/src/main/protos/api/api.proto b/protocol/src/main/protos/api/api.proto index 9a7534cf6ce..2505fa48d6f 100644 --- a/protocol/src/main/protos/api/api.proto +++ b/protocol/src/main/protos/api/api.proto @@ -1051,6 +1051,8 @@ message Return { SERVER_BUSY = 9; NO_CONNECTION = 10; NOT_ENOUGH_EFFECTIVE_CONNECTION = 11; + BLOCK_UNSOLIDIFIED = 12; + OTHER_ERROR = 20; } From a0f431cfe3783c56cf32bfc6071dbe2bde0e4180 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Fri, 29 Dec 2023 10:45:06 +0800 Subject: [PATCH 21/61] feat(net): delete config item node.discovery.bind.ip (#5597) * delete config item node.bind.ip * remove bindIp method from Args * get lan ip from libp2p --- common/build.gradle | 2 +- .../common/parameter/CommonParameter.java | 2 +- .../src/main/java/org/tron/core/Constant.java | 2 -- .../java/org/tron/core/config/args/Args.java | 30 ++++--------------- .../org/tron/core/config/args/ArgsTest.java | 19 +----------- .../core/services/RpcApiServicesTest.java | 6 ++-- .../LiteFnQueryGrpcInterceptorTest.java | 6 ++-- .../filter/RpcApiAccessInterceptorTest.java | 6 ++-- .../test/resources/config-test-mainnet.conf | 1 - .../resources/config-test-storagetest.conf | 1 - framework/src/test/resources/config-test.conf | 1 - 11 files changed, 17 insertions(+), 59 deletions(-) diff --git a/common/build.gradle b/common/build.gradle index 2cf15ec6efd..ba1c431e74c 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -46,7 +46,7 @@ dependencies { compile 'org.aspectj:aspectjrt:1.8.13' compile 'org.aspectj:aspectjweaver:1.8.13' compile 'org.aspectj:aspectjtools:1.8.13' - compile group: 'io.github.tronprotocol', name: 'libp2p', version: '2.1.0',{ + compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v2.2.1',{ exclude group: 'io.grpc', module: 'grpc-context' exclude group: 'io.grpc', module: 'grpc-core' exclude group: 'io.grpc', module: 'grpc-netty' diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index 1ad50e1e359..c42276e01bb 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -176,7 +176,7 @@ public class CommonParameter { public int nodeListenPort; @Getter @Setter - public String nodeDiscoveryBindIp; + public String nodeLanIp; @Getter @Setter public String nodeExternalIp; diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index d7ec3b715c9..f739358156f 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -281,8 +281,6 @@ public class Constant { public static final String EVENT_SUBSCRIBE_CONTRACT_ADDRESS = "event.subscribe.filter.contractAddress"; public static final String EVENT_SUBSCRIBE_CONTRACT_TOPIC = "event.subscribe.filter.contractTopic"; - public static final String NODE_DISCOVERY_BIND_IP = "node.discovery.bind.ip"; - public static final String NODE_DISCOVERY_EXTERNAL_IP = "node.discovery.external.ip"; public static final String NODE_BACKUP_PRIORITY = "node.backup.priority"; diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index aff6708dfff..b5f7d97726e 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -20,7 +20,6 @@ import java.io.InputStream; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.Socket; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; @@ -136,7 +135,7 @@ public static void clearParam() { PARAMETER.maxTps = 1000; PARAMETER.minParticipationRate = 0; PARAMETER.nodeListenPort = 0; - PARAMETER.nodeDiscoveryBindIp = ""; + PARAMETER.nodeLanIp = ""; PARAMETER.nodeExternalIp = ""; PARAMETER.nodeP2pVersion = 0; PARAMETER.nodeEnableIpv6 = false; @@ -635,7 +634,7 @@ public static void setParam(final String[] args, final String confFileName) { config.hasPath(Constant.NODE_LISTEN_PORT) ? config.getInt(Constant.NODE_LISTEN_PORT) : 0; - bindIp(config); + PARAMETER.nodeLanIp = PARAMETER.p2pConfig.getLanIp(); externalIp(config); PARAMETER.nodeP2pVersion = @@ -1247,7 +1246,7 @@ public static List getInetSocketAddress( if (filter) { String ip = inetSocketAddress.getAddress().getHostAddress(); int port = inetSocketAddress.getPort(); - if (!(PARAMETER.nodeDiscoveryBindIp.equals(ip) + if (!(PARAMETER.nodeLanIp.equals(ip) || PARAMETER.nodeExternalIp.equals(ip) || Constant.LOCAL_HOST.equals(ip)) || PARAMETER.nodeListenPort != port) { @@ -1525,25 +1524,6 @@ private static FilterQuery getEventFilter(final com.typesafe.config.Config confi return filter; } - private static void bindIp(final com.typesafe.config.Config config) { - if (!config.hasPath(Constant.NODE_DISCOVERY_BIND_IP) - || config.getString(Constant.NODE_DISCOVERY_BIND_IP) - .trim().isEmpty()) { - if (PARAMETER.nodeDiscoveryBindIp == null) { - logger.info("Bind address wasn't set, Punching to identify it..."); - try (Socket s = new Socket("www.baidu.com", 80)) { - PARAMETER.nodeDiscoveryBindIp = s.getLocalAddress().getHostAddress(); - logger.info("UDP local bound to: {}", PARAMETER.nodeDiscoveryBindIp); - } catch (IOException e) { - logger.warn("Can't get bind IP. Fall back to 127.0.0.1: " + e); - PARAMETER.nodeDiscoveryBindIp = "127.0.0.1"; - } - } - } else { - PARAMETER.nodeDiscoveryBindIp = config.getString(Constant.NODE_DISCOVERY_BIND_IP).trim(); - } - } - private static void externalIp(final com.typesafe.config.Config config) { if (!config.hasPath(Constant.NODE_DISCOVERY_EXTERNAL_IP) || config .getString(Constant.NODE_DISCOVERY_EXTERNAL_IP).trim().isEmpty()) { @@ -1551,7 +1531,7 @@ private static void externalIp(final com.typesafe.config.Config config) { logger.info("External IP wasn't set, using ipv4 from libp2p"); PARAMETER.nodeExternalIp = PARAMETER.p2pConfig.getIp(); if (StringUtils.isEmpty(PARAMETER.nodeExternalIp)) { - PARAMETER.nodeExternalIp = PARAMETER.nodeDiscoveryBindIp; + PARAMETER.nodeExternalIp = PARAMETER.nodeLanIp; } } } else { @@ -1622,7 +1602,7 @@ public static void logConfig() { logger.info("\n"); logger.info("************************ Net config ************************"); logger.info("P2P version: {}", parameter.getNodeP2pVersion()); - logger.info("Bind IP: {}", parameter.getNodeDiscoveryBindIp()); + logger.info("LAN IP: {}", parameter.getNodeLanIp()); logger.info("External IP: {}", parameter.getNodeExternalIp()); logger.info("Listen port: {}", parameter.getNodeListenPort()); logger.info("Node ipv6 enable: {}", parameter.isNodeEnableIpv6()); diff --git a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java index 13e272d1e13..56e418be7c1 100644 --- a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java +++ b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java @@ -17,11 +17,6 @@ import com.google.common.collect.Lists; import com.typesafe.config.Config; -import com.typesafe.config.ConfigMergeable; -import com.typesafe.config.ConfigOrigin; -import com.typesafe.config.ConfigRenderOptions; -import com.typesafe.config.ConfigValue; -import com.typesafe.config.ConfigValueType; import io.grpc.internal.GrpcUtil; import io.grpc.netty.NettyServerBuilder; import java.lang.reflect.InvocationTargetException; @@ -36,10 +31,8 @@ import org.tron.common.utils.ByteArray; import org.tron.common.utils.LocalWitnesses; import org.tron.common.utils.PublicMethod; -import org.tron.common.utils.ReflectUtils; import org.tron.core.Constant; import org.tron.core.config.Configuration; -import org.tron.core.net.peer.PeerManager; @Slf4j public class ArgsTest { @@ -95,7 +88,6 @@ public void get() { Assert.assertTrue(parameter.isNodeDiscoveryEnable()); Assert.assertTrue(parameter.isNodeDiscoveryPersist()); - Assert.assertEquals("127.0.0.1", parameter.getNodeDiscoveryBindIp()); Assert.assertEquals("46.168.1.1", parameter.getNodeExternalIp()); Assert.assertEquals(18888, parameter.getNodeListenPort()); Assert.assertEquals(2000, parameter.getNodeConnectionTimeout()); @@ -130,27 +122,18 @@ public void testIpFromLibP2p() Args.setParam(new String[] {"-w"}, Constant.TEST_CONF); CommonParameter parameter = Args.getInstance(); - String configuredBindIp = parameter.getNodeDiscoveryBindIp(); String configuredExternalIp = parameter.getNodeExternalIp(); - Assert.assertEquals("127.0.0.1", configuredBindIp); Assert.assertEquals("46.168.1.1", configuredExternalIp); Config config = Configuration.getByFileName(null, Constant.TEST_CONF); - Config config2 = config.withoutPath(Constant.NODE_DISCOVERY_BIND_IP); - Config config3 = config2.withoutPath(Constant.NODE_DISCOVERY_EXTERNAL_IP); + Config config3 = config.withoutPath(Constant.NODE_DISCOVERY_EXTERNAL_IP); - CommonParameter.getInstance().setNodeDiscoveryBindIp(null); CommonParameter.getInstance().setNodeExternalIp(null); - Method method1 = Args.class.getDeclaredMethod("bindIp", Config.class); - method1.setAccessible(true); - method1.invoke(Args.class, config3); - Method method2 = Args.class.getDeclaredMethod("externalIp", Config.class); method2.setAccessible(true); method2.invoke(Args.class, config3); - Assert.assertNotEquals(configuredBindIp, parameter.getNodeDiscoveryBindIp()); Assert.assertNotEquals(configuredExternalIp, parameter.getNodeExternalIp()); } } diff --git a/framework/src/test/java/org/tron/core/services/RpcApiServicesTest.java b/framework/src/test/java/org/tron/core/services/RpcApiServicesTest.java index 8b45b871af7..ebb9e0bb223 100644 --- a/framework/src/test/java/org/tron/core/services/RpcApiServicesTest.java +++ b/framework/src/test/java/org/tron/core/services/RpcApiServicesTest.java @@ -138,11 +138,11 @@ public static void init() throws IOException { getInstance().setRpcPort(PublicMethod.chooseRandomPort()); getInstance().setRpcOnSolidityPort(PublicMethod.chooseRandomPort()); getInstance().setRpcOnPBFTPort(PublicMethod.chooseRandomPort()); - String fullNode = String.format("%s:%d", getInstance().getNodeDiscoveryBindIp(), + String fullNode = String.format("%s:%d", getInstance().getNodeLanIp(), getInstance().getRpcPort()); - String solidityNode = String.format("%s:%d", getInstance().getNodeDiscoveryBindIp(), + String solidityNode = String.format("%s:%d", getInstance().getNodeLanIp(), getInstance().getRpcOnSolidityPort()); - String pBFTNode = String.format("%s:%d", getInstance().getNodeDiscoveryBindIp(), + String pBFTNode = String.format("%s:%d", getInstance().getNodeLanIp(), getInstance().getRpcOnPBFTPort()); ManagedChannel channelFull = ManagedChannelBuilder.forTarget(fullNode) diff --git a/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java b/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java index 00f2c2fc086..081b5839f84 100644 --- a/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java +++ b/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java @@ -58,11 +58,11 @@ public static void init() throws IOException { Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcOnSolidityPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcOnPBFTPort(PublicMethod.chooseRandomPort()); - String fullnode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), + String fullnode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), Args.getInstance().getRpcPort()); - String solidityNode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), + String solidityNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), Args.getInstance().getRpcOnSolidityPort()); - String pBFTNode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), + String pBFTNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), Args.getInstance().getRpcOnPBFTPort()); channelFull = ManagedChannelBuilder.forTarget(fullnode) .usePlaintext() diff --git a/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java b/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java index 7d95c0c368a..38af756f790 100644 --- a/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java +++ b/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java @@ -59,11 +59,11 @@ public static void init() throws IOException { Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcOnSolidityPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcOnPBFTPort(PublicMethod.chooseRandomPort()); - String fullNode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), + String fullNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), Args.getInstance().getRpcPort()); - String solidityNode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), + String solidityNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), Args.getInstance().getRpcOnSolidityPort()); - String pBFTNode = String.format("%s:%d", Args.getInstance().getNodeDiscoveryBindIp(), + String pBFTNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), Args.getInstance().getRpcOnPBFTPort()); ManagedChannel channelFull = ManagedChannelBuilder.forTarget(fullNode) diff --git a/framework/src/test/resources/config-test-mainnet.conf b/framework/src/test/resources/config-test-mainnet.conf index 43a01a0feb9..ad72d2afaaa 100644 --- a/framework/src/test/resources/config-test-mainnet.conf +++ b/framework/src/test/resources/config-test-mainnet.conf @@ -53,7 +53,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "127.0.0.1" external.ip = "46.168.1.1" } diff --git a/framework/src/test/resources/config-test-storagetest.conf b/framework/src/test/resources/config-test-storagetest.conf index 5098e39b650..d2dc27b795a 100644 --- a/framework/src/test/resources/config-test-storagetest.conf +++ b/framework/src/test/resources/config-test-storagetest.conf @@ -76,7 +76,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "127.0.0.1" external.ip = "46.168.1.1" } diff --git a/framework/src/test/resources/config-test.conf b/framework/src/test/resources/config-test.conf index 304ad125340..68a48ca64b2 100644 --- a/framework/src/test/resources/config-test.conf +++ b/framework/src/test/resources/config-test.conf @@ -74,7 +74,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "127.0.0.1" external.ip = "46.168.1.1" } From d518512e32fdc3ca857c2ef8d44ef9ccb169cd57 Mon Sep 17 00:00:00 2001 From: liukai Date: Tue, 2 Jan 2024 14:34:04 +0800 Subject: [PATCH 22/61] feat(LiteFullNode): remove LiteFullNodeTool --- framework/build.gradle | 6 - .../org/tron/tool/litefullnode/DbTool.java | 199 ------ .../tool/litefullnode/LiteFullNodeTool.java | 637 ------------------ .../java/org/tron/tool/litefullnode/README.md | 109 --- .../java/org/tron/tool/litefullnode/Util.java | 62 -- .../tool/litefullnode/db/DBInterface.java | 21 - .../tool/litefullnode/db/LevelDBImpl.java | 46 -- .../tool/litefullnode/db/RocksDBImpl.java | 66 -- .../litefullnode/iterator/DBIterator.java | 18 - .../iterator/LevelDBIterator.java | 47 -- .../litefullnode/iterator/RockDBIterator.java | 48 -- .../tron/program/LiteFullNodeToolTest.java | 202 ------ plugins/README.md | 5 - 13 files changed, 1466 deletions(-) delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/DbTool.java delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/README.md delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/Util.java delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/db/DBInterface.java delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/db/LevelDBImpl.java delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/db/RocksDBImpl.java delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/iterator/DBIterator.java delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/iterator/LevelDBIterator.java delete mode 100644 framework/src/main/java/org/tron/tool/litefullnode/iterator/RockDBIterator.java delete mode 100644 framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java diff --git a/framework/build.gradle b/framework/build.gradle index c894ae1e03d..08e2e88e826 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -239,13 +239,11 @@ createScript(project, 'org.tron.program.SolidityNode', 'SolidityNode') createScript(project, 'org.tron.program.FullNode', 'FullNode') createScript(project, 'org.tron.program.KeystoreFactory', 'KeystoreFactory') createScript(project, 'org.tron.program.DBConvert', 'DBConvert') -createScript(project, 'org.tron.tool.litefullnode.LiteFullNodeTool', 'LiteFullNodeTool') def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true' def skipSolidity = hasProperty('skipSolidity') ? true : false def skipKeystore = hasProperty('skipKeystore') ? true : false def skipConvert = hasProperty('skipConvert') ? true : false -def skipLite = hasProperty('skipLite') ? true : false def skipAll = hasProperty('skipAll') ? true : false if (releaseBinary == 'true') { artifacts { @@ -264,10 +262,6 @@ if (releaseBinary == 'true') { artifacts { archives(binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert'))} } - if (!skipLite) { - artifacts { - archives(binaryRelease('buildLiteFullNodeToolJar', 'LiteFullNodeTool', 'org.tron.tool.litefullnode.LiteFullNodeTool'))} - } } } diff --git a/framework/src/main/java/org/tron/tool/litefullnode/DbTool.java b/framework/src/main/java/org/tron/tool/litefullnode/DbTool.java deleted file mode 100644 index 5f43361eddf..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/DbTool.java +++ /dev/null @@ -1,199 +0,0 @@ -package org.tron.tool.litefullnode; - -import static org.fusesource.leveldbjni.JniDBFactory.factory; - -import com.google.common.collect.Maps; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Iterator; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.iq80.leveldb.CompressionType; -import org.iq80.leveldb.DB; -import org.iq80.leveldb.DBIterator; -import org.iq80.leveldb.Options; -import org.iq80.leveldb.WriteOptions; -import org.rocksdb.BlockBasedTableConfig; -import org.rocksdb.BloomFilter; -import org.rocksdb.RocksDBException; -import org.tron.common.utils.PropUtil; -import org.tron.tool.litefullnode.db.DBInterface; -import org.tron.tool.litefullnode.db.LevelDBImpl; -import org.tron.tool.litefullnode.db.RocksDBImpl; - -@Slf4j(topic = "tool") -public class DbTool { - - private static final String KEY_ENGINE = "ENGINE"; - public static final String ENGINE_FILE = "engine.properties"; - private static final String FILE_SEPARATOR = File.separator; - private static final String ROCKSDB = "ROCKSDB"; - - private static Map dbMap = Maps.newHashMap(); - - enum DbType { - LevelDB, - RocksDB - } - - /** - * Get the DB object according to the specified path, - * create db object when not exists, otherwise get it from the dbMap. - * - * @param sourceDir the parent path of db - * @param dbName db dir name - * - * @return db object - * - * @throws IOException IOException - * @throws RocksDBException RocksDBException - */ - public static DBInterface getDB(String sourceDir, String dbName) - throws IOException, RocksDBException { - Path path = Paths.get(sourceDir, dbName); - if (dbMap.containsKey(path.toString())) { - return dbMap.get(path.toString()); - } - DbType type = getDbType(sourceDir, dbName); - DBInterface db; - switch (type) { - case LevelDB: - db = openLevelDb(path); - dbMap.put(path.toString(), db); - break; - case RocksDB: - db = openRocksDb(path); - dbMap.put(path.toString(), db); - break; - default: - throw new IllegalStateException("Unexpected value: " + type); - } - return db; - } - - /** - * Close db. - * @param sourceDir db parentPath - * @param dbName db dirname - * @throws IOException IOException - */ - public static void closeDB(String sourceDir, String dbName) - throws IOException { - Path path = Paths.get(sourceDir, dbName); - DBInterface db = dbMap.get(path.toString()); - if (db != null) { - try { - dbMap.remove(path.toString()); - db.close(); - } catch (IOException e) { - logger.error("close db {} error: {}", path, e); - throw e; - } - } - } - - /** - * Close all dbs. - */ - public static void close() { - Iterator> iterator = dbMap.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry next = iterator.next(); - try { - next.getValue().close(); - } catch (IOException e) { - logger.error("close db failed, db: {}", next.getKey(), e); - } - iterator.remove(); - } - } - - private static DbType getDbType(String sourceDir, String dbName) { - String engineFile = String.format("%s%s%s%s%s", sourceDir, FILE_SEPARATOR, - dbName, FILE_SEPARATOR, ENGINE_FILE); - if (!new File(engineFile).exists()) { - return DbType.LevelDB; - } - String engine = PropUtil.readProperty(engineFile, KEY_ENGINE); - if (engine.equalsIgnoreCase(ROCKSDB)) { - return DbType.RocksDB; - } else { - return DbType.LevelDB; - } - } - - private static LevelDBImpl openLevelDb(Path db) throws IOException { - DB database; - Options options = getLevelDbOptions(); - try { - database = factory.open(db.toFile(), options); - } catch (IOException e) { - if (e.getMessage().contains("Corruption:")) { - factory.repair(db.toFile(), options); - database = factory.open(db.toFile(), options); - } else { - throw e; - } - } - return new LevelDBImpl(database); - } - - private static RocksDBImpl openRocksDb(Path db) throws RocksDBException { - org.rocksdb.RocksDB database; - try (org.rocksdb.Options options = newDefaultRocksDbOptions()) { - database = org.rocksdb.RocksDB.open(options, db.toString()); - } catch (Exception e) { - throw e; - } - return new RocksDBImpl(database); - } - - private static org.rocksdb.Options newDefaultRocksDbOptions() { - org.rocksdb.Options options = new org.rocksdb.Options(); - - options.setCreateIfMissing(true); - options.setIncreaseParallelism(1); - options.setNumLevels(7); - options.setMaxOpenFiles(-1); - options.setTargetFileSizeBase(64 * 1024 * 1024); - options.setTargetFileSizeMultiplier(1); - options.setMaxBytesForLevelBase(512 * 1024 * 1024); - options.setMaxBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); - options.setLevel0FileNumCompactionTrigger(4); - options.setLevelCompactionDynamicLevelBytes(true); - - BlockBasedTableConfig tableCfg = new BlockBasedTableConfig(); - tableCfg.setBlockSize(64 * 1024); - tableCfg.setBlockCacheSize(32 * 1024 * 1024); - tableCfg.setCacheIndexAndFilterBlocks(true); - tableCfg.setPinL0FilterAndIndexBlocksInCache(true); - tableCfg.setFilter(new BloomFilter(10, false)); - - options.setTableFormatConfig(tableCfg); - return options; - } - - private static Options getLevelDbOptions() { - CompressionType defaultCompressionType = CompressionType.SNAPPY; - int defaultBlockSize = 4 * 1024; - int defaultWriteBufferSize = 10 * 1024 * 1024; - long defaultCacheSize = 10 * 1024 * 1024L; - int defaultMaxOpenFiles = 100; - - Options dbOptions = new Options(); - - dbOptions.createIfMissing(true); - dbOptions.paranoidChecks(true); - dbOptions.verifyChecksums(true); - - dbOptions.compressionType(defaultCompressionType); - dbOptions.blockSize(defaultBlockSize); - dbOptions.writeBufferSize(defaultWriteBufferSize); - dbOptions.cacheSize(defaultCacheSize); - dbOptions.maxOpenFiles(defaultMaxOpenFiles); - - return dbOptions; - } -} diff --git a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java deleted file mode 100644 index 40c371c58e0..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java +++ /dev/null @@ -1,637 +0,0 @@ -package org.tron.tool.litefullnode; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.internal.Lists; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Strings; -import com.google.common.collect.Maps; -import com.google.common.primitives.Bytes; -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.rocksdb.RocksDBException; -import org.tron.common.parameter.CommonParameter; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.FileUtil; -import org.tron.common.utils.PropUtil; -import org.tron.core.Constant; -import org.tron.core.capsule.BlockCapsule; -import org.tron.core.capsule.TransactionCapsule; -import org.tron.core.db2.common.Value; -import org.tron.core.db2.core.SnapshotManager; -import org.tron.core.exception.BadItemException; -import org.tron.tool.litefullnode.db.DBInterface; -import org.tron.tool.litefullnode.iterator.DBIterator; - -@Slf4j(topic = "tool") -@Deprecated -public class LiteFullNodeTool { - - private static final long START_TIME = System.currentTimeMillis() / 1000; - - private static long RECENT_BLKS = 65536; - - private static final String SNAPSHOT_DIR_NAME = "snapshot"; - private static final String HISTORY_DIR_NAME = "history"; - private static final String INFO_FILE_NAME = "info.properties"; - private static final String BACKUP_DIR_PREFIX = ".bak_"; - private static final String CHECKPOINT_DB = "tmp"; - private static final String CHECKPOINT_DB_V2 = "checkpoint"; - private static final String BLOCK_DB_NAME = "block"; - private static final String BLOCK_INDEX_DB_NAME = "block-index"; - private static final String TRANS_DB_NAME = "trans"; - private static final String TRANSACTION_RET_DB_NAME = "transactionRetStore"; - private static final String TRANSACTION_HISTORY_DB_NAME = "transactionHistoryStore"; - private static final String PROPERTIES_DB_NAME = "properties"; - - private static final String DIR_FORMAT_STRING = "%s%s%s"; - - private static List archiveDbs = Arrays.asList( - BLOCK_DB_NAME, - BLOCK_INDEX_DB_NAME, - TRANS_DB_NAME, - TRANSACTION_RET_DB_NAME, - TRANSACTION_HISTORY_DB_NAME); - - /** - * Create the snapshot dataset. - * - * @param sourceDir the original fullnode database dir, - * same with {storage.db.directory} in conf file. - * @param snapshotDir the path that stores the snapshot dataset - */ - public void generateSnapshot(String sourceDir, String snapshotDir) { - logger.info("Start create snapshot."); - long start = System.currentTimeMillis(); - snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString(); - try { - hasEnoughBlock(sourceDir); - List snapshotDbs = getSnapshotDbs(sourceDir); - split(sourceDir, snapshotDir, snapshotDbs); - mergeCheckpoint2Snapshot(sourceDir, snapshotDir); - // write genesisBlock , latest recent blocks and trans - fillSnapshotBlockAndTransDb(sourceDir, snapshotDir); - generateInfoProperties(Paths.get(snapshotDir, INFO_FILE_NAME).toString(), sourceDir); - } catch (IOException | RocksDBException e) { - logger.error("Create snapshot failed, {}.", e.getMessage()); - return; - } - long end = System.currentTimeMillis(); - logger.info("Create snapshot finished, take {} s.\n", (end - start) / 1000); - } - - /** - * Create the history dataset. - * - * @param sourceDir the original fullnode database dir, - * same with {storage.db.directory} in conf file. - * @param historyDir the path that stores the history dataset - */ - public void generateHistory(String sourceDir, String historyDir) { - logger.info("Start create history."); - long start = System.currentTimeMillis(); - historyDir = Paths.get(historyDir, HISTORY_DIR_NAME).toString(); - try { - if (isLite(sourceDir)) { - throw new IllegalStateException( - String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir)); - } - hasEnoughBlock(sourceDir); - split(sourceDir, historyDir, archiveDbs); - mergeCheckpoint2History(sourceDir, historyDir); - generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(), sourceDir); - } catch (IOException | RocksDBException e) { - logger.error("Create history failed, {}.", e.getMessage()); - return; - } - long end = System.currentTimeMillis(); - logger.info("Create history finished, take {} s.\n", (end - start) / 1000); - } - - /** - * Merge the history dataset into database. - * - * @param historyDir the path that stores the history dataset - * - * @param databaseDir lite fullnode database path - */ - public void completeHistoryData(String historyDir, String databaseDir) { - logger.info("Start merge history to lite node."); - long start = System.currentTimeMillis(); - BlockNumInfo blockNumInfo = null; - try { - // check historyDir is from lite data - if (isLite(historyDir)) { - throw new IllegalStateException( - String.format("Unavailable history: %s is not generated by fullNode data.", - historyDir)); - } - // 1. check block number and genesis block are compatible, - // and return the block numbers of snapshot and history - blockNumInfo = checkAndGetBlockNumInfo(historyDir, databaseDir); - // 2. move archive dbs to bak - backupArchiveDbs(databaseDir); - // 3. copy history data to databaseDir - copyHistory2Database(historyDir, databaseDir); - // 4. delete the duplicate block data in history data - trimHistory(databaseDir, blockNumInfo); - // 5. merge bak to database - mergeBak2Database(databaseDir); - // 6. delete snapshot flag - deleteSnapshotFlag(databaseDir); - } catch (IOException | RocksDBException | BadItemException e) { - logger.error("Merge history data to database failed, {}.", e.getMessage()); - return; - } - long end = System.currentTimeMillis(); - logger.info("Merge history finished, take {} s. \n", (end - start) / 1000); - } - - private List getSnapshotDbs(String sourceDir) { - List snapshotDbs = Lists.newArrayList(); - File basePath = new File(sourceDir); - Arrays.stream(Objects.requireNonNull(basePath.listFiles())) - .filter(File::isDirectory) - .filter(dir -> !archiveDbs.contains(dir.getName())) - .forEach(dir -> snapshotDbs.add(dir.getName())); - return snapshotDbs; - } - - private void mergeCheckpoint2Snapshot(String sourceDir, String historyDir) { - List snapshotDbs = getSnapshotDbs(sourceDir); - mergeCheckpoint(sourceDir, historyDir, snapshotDbs); - } - - private void mergeCheckpoint2History(String sourceDir, String destDir) { - mergeCheckpoint(sourceDir, destDir, archiveDbs); - } - - private void split(String sourceDir, String destDir, List dbs) throws IOException { - logger.info("Begin to split the dbs."); - if (!new File(sourceDir).isDirectory()) { - throw new RuntimeException(String.format("sourceDir: %s must be a directory ", sourceDir)); - } - File destPath = new File(destDir); - if (new File(destDir).exists()) { - throw new RuntimeException(String.format( - "destDir: %s is already exist, please remove it first", destDir)); - } - if (!destPath.mkdirs()) { - throw new RuntimeException(String.format("destDir: %s create failed, please check", destDir)); - } - Util.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs); - } - - private void mergeCheckpoint(String sourceDir, String destDir, List destDbs) { - logger.info("Begin to merge checkpoint to dataset."); - try { - List cpList = getCheckpointV2List(sourceDir); - if (cpList.size() > 0) { - for (String cp: cpList) { - DBInterface checkpointDb = DbTool.getDB(sourceDir + "/" + CHECKPOINT_DB_V2, cp); - recover(checkpointDb, destDir, destDbs); - } - } else { - DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); - recover(tmpDb, destDir, destDbs); - } - } catch (IOException | RocksDBException e) { - throw new RuntimeException(e); - } - } - - private void recover(DBInterface db, String destDir, List destDbs) - throws IOException, RocksDBException { - try (DBIterator iterator = db.iterator()) { - for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { - byte[] key = iterator.getKey(); - byte[] value = iterator.getValue(); - String dbName = SnapshotManager.simpleDecode(key); - byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length); - byte[] realValue = - value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); - if (destDbs != null && destDbs.contains(dbName)) { - DBInterface destDb = DbTool.getDB(destDir, dbName); - if (realValue != null) { - destDb.put(realKey, realValue); - } else { - byte op = value[0]; - if (Value.Operator.DELETE.getValue() == op) { - destDb.delete(realKey); - } else { - destDb.put(realKey, new byte[0]); - } - } - } - } - } - } - - private void generateInfoProperties(String propertyfile, String databaseDir) - throws IOException, RocksDBException { - logger.info("Create {} for dataset.", INFO_FILE_NAME); - if (!FileUtil.createFileIfNotExists(propertyfile)) { - throw new RuntimeException("Create properties file failed."); - } - if (!PropUtil.writeProperty(propertyfile, Constant.SPLIT_BLOCK_NUM, - Long.toString(getLatestBlockHeaderNum(databaseDir)))) { - throw new RuntimeException("Write properties file failed."); - } - } - - private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException { - // query latest_block_header_number from checkpoint first - final String latestBlockHeaderNumber = "latest_block_header_number"; - List cpList = getCheckpointV2List(databaseDir); - DBInterface checkpointDb = null; - if (cpList.size() > 0) { - String lastestCp = cpList.get(cpList.size() - 1); - checkpointDb = DbTool.getDB(databaseDir + "/" + CHECKPOINT_DB_V2, lastestCp); - } else { - checkpointDb = DbTool.getDB(databaseDir, CHECKPOINT_DB); - } - Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb, - latestBlockHeaderNumber.getBytes()); - if (blockNumber != null) { - return blockNumber; - } - // query from propertiesDb if checkpoint not contains latest_block_header_number - DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME); - return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber))) - .map(ByteArray::toLong) - .orElseThrow( - () -> new IllegalArgumentException("not found latest block header number")); - } - - private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) { - byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key)); - if (value != null && value.length > 1) { - return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length)); - } - return null; - } - - /** - * recent blocks, trans and genesis block. - */ - private void fillSnapshotBlockAndTransDb(String sourceDir, String snapshotDir) - throws IOException, RocksDBException { - logger.info("Begin to fill {} block, genesis block and trans to snapshot.", RECENT_BLKS); - DBInterface sourceBlockIndexDb = DbTool.getDB(sourceDir, BLOCK_INDEX_DB_NAME); - DBInterface sourceBlockDb = DbTool.getDB(sourceDir, BLOCK_DB_NAME); - DBInterface destBlockDb = DbTool.getDB(snapshotDir, BLOCK_DB_NAME); - DBInterface destBlockIndexDb = DbTool.getDB(snapshotDir, BLOCK_INDEX_DB_NAME); - DBInterface destTransDb = DbTool.getDB(snapshotDir, TRANS_DB_NAME); - // put genesis block and block-index into snapshot - long genesisBlockNum = 0L; - byte[] genesisBlockID = sourceBlockIndexDb.get(ByteArray.fromLong(genesisBlockNum)); - destBlockIndexDb.put(ByteArray.fromLong(genesisBlockNum), genesisBlockID); - destBlockDb.put(genesisBlockID, sourceBlockDb.get(genesisBlockID)); - - long latestBlockNum = getLatestBlockHeaderNum(sourceDir); - long startIndex = latestBlockNum - RECENT_BLKS + 1; - // put the recent blocks and trans in snapshot - for (long blockNum = startIndex; blockNum <= latestBlockNum; blockNum++) { - try { - byte[] blockId = getDataFromSourceDB(sourceDir, BLOCK_INDEX_DB_NAME, - Longs.toByteArray(blockNum)); - byte[] block = getDataFromSourceDB(sourceDir, BLOCK_DB_NAME, blockId); - // put block - destBlockDb.put(blockId, block); - // put block index - destBlockIndexDb.put(ByteArray.fromLong(blockNum), blockId); - // put trans - long finalBlockNum = blockNum; - new BlockCapsule(block).getTransactions().stream().map( - tc -> tc.getTransactionId().getBytes()) - .map(bytes -> Maps.immutableEntry(bytes, Longs.toByteArray(finalBlockNum))) - .forEach(e -> destTransDb.put(e.getKey(), e.getValue())); - } catch (IOException | RocksDBException | BadItemException e) { - throw new RuntimeException(e.getMessage()); - } - } - // copy engine.properties for block、block-index、trans from source if exist - copyEngineIfExist(sourceDir, snapshotDir, BLOCK_DB_NAME, BLOCK_INDEX_DB_NAME, TRANS_DB_NAME); - } - - private void copyEngineIfExist(String source, String dest, String... dbNames) { - for (String dbName : dbNames) { - Path ori = Paths.get(source, dbName, DbTool.ENGINE_FILE); - if (ori.toFile().exists()) { - Util.copy(ori, Paths.get(dest, dbName, DbTool.ENGINE_FILE)); - } - } - } - - private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDBException { - long genesisBlockNum = 0L; - DBInterface blockIndexDb = DbTool.getDB(parentDir, BLOCK_INDEX_DB_NAME); - byte[] result = blockIndexDb.get(ByteArray.fromLong(genesisBlockNum)); - // when merge history, block-index db will be moved to bak dir and replaced by history - // so should close this db and reopen it. - DbTool.closeDB(parentDir, BLOCK_INDEX_DB_NAME); - return result; - } - - private static byte[] simpleEncode(String s) { - byte[] bytes = s.getBytes(); - byte[] length = Ints.toByteArray(bytes.length); - byte[] r = new byte[4 + bytes.length]; - System.arraycopy(length, 0, r, 0, 4); - System.arraycopy(bytes, 0, r, 4, bytes.length); - return r; - } - - private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String databaseDir) - throws IOException, RocksDBException { - logger.info("Check the compatibility of this history."); - String snapshotInfo = String.format( - DIR_FORMAT_STRING, databaseDir, File.separator, INFO_FILE_NAME); - String historyInfo = String.format( - DIR_FORMAT_STRING, historyDir, File.separator, INFO_FILE_NAME); - if (!FileUtil.isExists(snapshotInfo)) { - throw new FileNotFoundException( - "Snapshot property file is not found. maybe this is a complete fullnode?"); - } - if (!FileUtil.isExists(historyInfo)) { - throw new FileNotFoundException("history property file is not found."); - } - long snapshotBlkNum = Long.parseLong(PropUtil.readProperty(snapshotInfo, Constant - .SPLIT_BLOCK_NUM)); - long historyBlkNum = Long.parseLong(PropUtil.readProperty(historyInfo, Constant - .SPLIT_BLOCK_NUM)); - if (historyBlkNum < snapshotBlkNum) { - throw new RuntimeException( - String.format( - "History latest block number is lower than snapshot, history: %d, snapshot: %d", - historyBlkNum, snapshotBlkNum)); - } - // check genesis block is equal - if (!Arrays.equals(getGenesisBlockHash(databaseDir), getGenesisBlockHash(historyDir))) { - throw new RuntimeException(String.format( - "Genesis block hash is not equal, history: %s, database: %s", - Arrays.toString(getGenesisBlockHash(historyDir)), - Arrays.toString(getGenesisBlockHash(databaseDir)))); - } - return new BlockNumInfo(snapshotBlkNum, historyBlkNum); - } - - private void backupArchiveDbs(String databaseDir) throws IOException { - String bakDir = String.format("%s%s%s%d", - databaseDir, File.separator, BACKUP_DIR_PREFIX, START_TIME); - logger.info("Backup the archive dbs to {}.", bakDir); - if (!FileUtil.createDirIfNotExists(bakDir)) { - throw new RuntimeException(String.format("create bak dir %s failed", bakDir)); - } - Util.copyDatabases(Paths.get(databaseDir), Paths.get(bakDir), archiveDbs); - archiveDbs.forEach(db -> FileUtil.deleteDir(new File(databaseDir, db))); - } - - private void copyHistory2Database(String historyDir, String databaseDir) throws IOException { - logger.info("Begin to copy history to database."); - Util.copyDatabases(Paths.get(historyDir), Paths.get(databaseDir), archiveDbs); - } - - private void trimHistory(String databaseDir, BlockNumInfo blockNumInfo) - throws BadItemException, IOException, RocksDBException { - logger.info("Begin to trim the history data."); - DBInterface blockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); - DBInterface blockDb = DbTool.getDB(databaseDir, BLOCK_DB_NAME); - DBInterface transDb = DbTool.getDB(databaseDir, TRANS_DB_NAME); - DBInterface tranRetDb = DbTool.getDB(databaseDir, TRANSACTION_RET_DB_NAME); - for (long n = blockNumInfo.getHistoryBlkNum(); n > blockNumInfo.getSnapshotBlkNum(); n--) { - byte[] blockIdHash = blockIndexDb.get(ByteArray.fromLong(n)); - BlockCapsule block = new BlockCapsule(blockDb.get(blockIdHash)); - // delete transactions - for (TransactionCapsule e : block.getTransactions()) { - transDb.delete(e.getTransactionId().getBytes()); - } - // delete transaction result - tranRetDb.delete(ByteArray.fromLong(n)); - // delete block - blockDb.delete(blockIdHash); - // delete block index - blockIndexDb.delete(ByteArray.fromLong(n)); - } - } - - private void mergeBak2Database(String databaseDir) throws IOException, RocksDBException { - String bakDir = String.format("%s%s%s%d", - databaseDir, File.separator, BACKUP_DIR_PREFIX, START_TIME); - logger.info("Begin to merge {} to database.", bakDir); - for (String dbName : archiveDbs) { - DBInterface bakDb = DbTool.getDB(bakDir, dbName); - DBInterface destDb = DbTool.getDB(databaseDir, dbName); - try (DBIterator iterator = bakDb.iterator()) { - for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { - destDb.put(iterator.getKey(), iterator.getValue()); - } - } - } - } - - private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key) - throws IOException, RocksDBException { - DBInterface sourceDb = DbTool.getDB(sourceDir, dbName); - DBInterface checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); - // get data from tmp first. - byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key)); - byte[] value; - if (isEmptyBytes(valueFromTmp)) { - value = sourceDb.get(key); - } else { - value = valueFromTmp.length == 1 - ? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length); - } - if (isEmptyBytes(value)) { - throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s", - dbName, Arrays.toString(key))); - } - return value; - } - - /** - * return true if byte array is null or length is 0. - * @param b bytes - * @return true or false - */ - private static boolean isEmptyBytes(byte[] b) { - if (b != null) { - return b.length == 0; - } - return true; - } - - private void deleteSnapshotFlag(String databaseDir) throws IOException, RocksDBException { - logger.info("Delete the info file from {}.", databaseDir); - Files.delete(Paths.get(databaseDir, INFO_FILE_NAME)); - } - - private void hasEnoughBlock(String sourceDir) throws RocksDBException, IOException { - // check latest - long latest = getLatestBlockHeaderNum(sourceDir); - // check second ,skip 0; - long second = getSecondBlock(sourceDir); - if (latest - second + 1 < RECENT_BLKS) { - throw new NoSuchElementException( - String.format("At least %d blocks in block store, actual latestBlock:%d, firstBlock:%d.", - RECENT_BLKS, latest, second)); - } - } - - private boolean isLite(String databaseDir) throws RocksDBException, IOException { - return getSecondBlock(databaseDir) > 1; - } - - private long getSecondBlock(String databaseDir) throws RocksDBException, IOException { - long num = 0; - DBInterface sourceBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); - DBIterator iterator = sourceBlockIndexDb.iterator(); - iterator.seek(ByteArray.fromLong(1)); - if (iterator.hasNext()) { - num = Longs.fromByteArray(iterator.getKey()); - } - return num; - } - - @VisibleForTesting - public static void setRecentBlks(long recentBlks) { - RECENT_BLKS = recentBlks; - } - - @VisibleForTesting - public static void reSetRecentBlks() { - RECENT_BLKS = 65536; - } - - private List getCheckpointV2List(String sourceDir) { - File file = new File(Paths.get(sourceDir, CHECKPOINT_DB_V2).toString()); - if (file.exists() && file.isDirectory() && file.list() != null) { - return Arrays.stream(file.list()).sorted().collect(Collectors.toList()); - } - return Lists.newArrayList(); - } - - private void run(Args argv) { - if (StringUtils.isBlank(argv.fnDataPath) || StringUtils.isBlank(argv.datasetPath)) { - throw new ParameterException("fnDataPath or datasetPath can't be null"); - } - switch (argv.operate) { - case "split": - if (Strings.isNullOrEmpty(argv.type)) { - throw new ParameterException("type can't be null when operate=split"); - } - if (SNAPSHOT_DIR_NAME.equals(argv.type)) { - generateSnapshot(argv.fnDataPath, argv.datasetPath); - } else if (HISTORY_DIR_NAME.equals(argv.type)) { - generateHistory(argv.fnDataPath, argv.datasetPath); - } else { - throw new ParameterException("not support type:" + argv.type); - } - break; - case "merge": - completeHistoryData(argv.datasetPath, argv.fnDataPath); - break; - default: - throw new ParameterException("not supportted operate:" + argv.operate); - } - DbTool.close(); - } - - /** - * main. - */ - public static void main(String[] args) { - logger.info("LiteFullNodeTool is deprecated and it will be removed in the next major release," - + " use Toolkit.jar db lite instead, parameters are fully compatible."); - Args argv = new Args(); - CommonParameter.getInstance().setValidContractProtoThreadNum(1); - LiteFullNodeTool tool = new LiteFullNodeTool(); - JCommander jct = JCommander.newBuilder() - .addObject(argv) - .build(); - jct.setProgramName("lite fullnode tool"); - try { - jct.parse(args); - if (argv.help) { - jct.usage(); - } else { - tool.run(argv); - } - } catch (Exception e) { - logger.error(e.getMessage()); - jct.usage(); - } - } - - static class Args { - @Parameter( - names = {"--operate", "-o"}, - help = true, required = true, - description = "operate: [ split | merge ]", - order = 1) - private String operate; - @Parameter(names = {"--type", "-t"}, - help = true, - description = "only used with operate=split: [ snapshot | history ]", - order = 2) - private String type; - @Parameter( - names = {"--fn-data-path"}, - help = true, required = true, - description = "the fullnode database path," - + " defined as ${storage.db.directory} in config.conf", - order = 3) - private String fnDataPath; - @Parameter( - names = {"--dataset-path"}, - help = true, required = true, - description = "dataset directory, when operation is `split`, " - + "`dataset-path` is the path that store the `Snapshot Dataset` or " - + "`History Dataset`, otherwise `dataset-path` should be " - + "the `History Dataset` path", - order = 4) - private String datasetPath; - @Parameter( - names = "--help", - help = true, - order = 5) - private boolean help; - } - - static class BlockNumInfo { - private long snapshotBlkNum; - private long historyBlkNum; - - public BlockNumInfo(long snapshotBlkNum, long historyBlkNum) { - this.snapshotBlkNum = snapshotBlkNum; - this.historyBlkNum = historyBlkNum; - } - - public long getSnapshotBlkNum() { - return snapshotBlkNum; - } - - public long getHistoryBlkNum() { - return historyBlkNum; - } - } -} - - - diff --git a/framework/src/main/java/org/tron/tool/litefullnode/README.md b/framework/src/main/java/org/tron/tool/litefullnode/README.md deleted file mode 100644 index 24357c2a99e..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# Lite FullNode Tool - -## Introduction - -Lite FullNode Tool is used to split the database of a FullNode into a `Snapshot dataset` and a `History dataset`. - -- `Snapshot dataset`: the minimum dataset for quick startup of the Lite FullNode. -- `History dataset`: the archive dataset that used for historical data queries. - -Remember stop the FullNode process before any operation. This tool provides the ability to specify which dataset to split. -The two datasets are split by the `latest_block_number`. Lite FullNode that startup by `Snapshot dataset` does not support query the historical data behind the `latest_block_number`, -this tool also provides a merge function that can merge `History dataset` into the database of Lite FullNode. For more API details: [HTTP&GRPC APIs](#HTTP&GRPC-APIs) - -For more design details, please refer to: [TIP128](https://github.com/tronprotocol/tips/issues/128) - -## Usage - -### Options - -This tool provides independent cutting of `Snapshot Dataset` and `History Dataset` and a merge function. - -- `--operation | -o`: [ split | merge ] specifies the operation as either to split or to merge -- `--type | -t`: [ snapshot | history ] is used only with `split` to specify the type of the dataset to be split; snapshot refers to Snapshot Dataset and history refers to History Dataset. -- `--fn-data-path`: FullNode database directory -- `--dataset-path`: dataset directory, when operation is `split`, `dataset-path` is the path that store the `Snapshot Dataset` or `History Dataset`, -otherwise `dataset-path` should be the `History Dataset` path. - -### Example - -Start a new FullNode using the default config, then an `output-directory` will be produced in the current directory. -`output-directory` contains a sub-directory named `database` which is the database to be split. - -#### Split and get a `Snapshot` - -First, stop the FullNode and execute: -``` -// just for simplify, locate the snapshot into `/tmp` directory, -java -jar LiteFullNodeTool.jar -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp -``` -then a `snapshot` directory will be generated in `/tmp`, pack this directory and copy it to somewhere that is ready to run a Lite Fullnode. -Do not forget rename the directory from `snapshot` to `database`. -(the default value of the storage.db.directory is `database`, make sure rename the snapshot to the specified value) - -#### Split a `History` - -If historical data query is needed, `History dataset` should be generated and merged into Lite FullNode. -``` -// just for simplify, locate the history into `/tmp` directory, -java -jar LiteFullNodeTool.jar -o split -t history --fn-data-path output-directory/database --dataset-path /tmp -``` -A `history` directory will be generated in `/tmp`, pack this directory and copy it to a Lite Fullnode. -`History dataset` always take a large storage, make sure the disk has enough volume to store the `History dataset`. - -#### Merge - -Both `History Dataset` and `Snapshot Dataset` have an info.properties file to identify the block height from which they are segmented. -Make sure that the `split_block_num` in `History Dataset` is not less than the corresponding value in the `Snapshot Dataset`. - -After getting the `History dataset`, the Lite FullNode can merge the `History dataset` and become a real FullNode. -``` -// just for simplify, assume `History dataset` is locate in /tmp -java -jar LiteFullNodeTool.jar -o merge --fn-data-path output-directory/database --dataset-path /tmp/history -``` - -### HTTP&GRPC APIs - -Some APIs are not supported on lite fullnode, here is the list: - -#### Http - -| wallet/ | walletsolidity/ | -|---|---| -| getblockbyid | getblockbyid | -| getblockbylatestnum | getblockbylatestnum | -| getblockbylimitnext | getblockbylimitnext | -| getblockbynum | getblockbynum | -| getmerkletreevoucherinfo | getmerkletreevoucherinfo | -| gettransactionbyid | gettransactionbyid | -| gettransactioncountbyblocknum | gettransactioncountbyblocknum | -| gettransactioninfobyid | gettransactioninfobyid | -| gettransactionreceiptbyid | | -| isspend | isspend | -| scanandmarknotebyivk | scanandmarknotebyivk | -| scannotebyivk | scannotebyivk | -| scannotebyovk | scannotebyovk | -| totaltransaction | | - -#### GRPC - -| protocol.Wallet | protocol.WalletSolidity | protocol.Database | -|---|---|---| -| GetBlockById | | | -| GetBlockByLatestNum | | | -| GetBlockByLatestNum2 | | | -| GetBlockByLimitNext | | | -| GetBlockByLimitNext2 | | | -| GetBlockByNum | GetBlockByNum | GetBlockByNum | -| GetBlockByNum2 | GetBlockByNum2 | | -| GetMerkleTreeVoucherInfo | GetMerkleTreeVoucherInfo | | -| GetTransactionById | GetTransactionById | | -| GetTransactionCountByBlockNum | GetTransactionCountByBlockNum | | -| GetTransactionInfoById | GetTransactionInfoById | | -| IsSpend | IsSpend | | -| ScanAndMarkNoteByIvk | ScanAndMarkNoteByIvk | | -| ScanNoteByIvk | ScanNoteByIvk | | -| ScanNoteByOvk | ScanNoteByOvk | | -| TotalTransaction | | | - -These APIs can open forcibly by set `openHistoryQueryWhenLiteFN` = true, but not recommended. \ No newline at end of file diff --git a/framework/src/main/java/org/tron/tool/litefullnode/Util.java b/framework/src/main/java/org/tron/tool/litefullnode/Util.java deleted file mode 100644 index 0e4898b8031..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/Util.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.tron.tool.litefullnode; - -import java.io.IOException; -import java.nio.file.FileSystemException; -import java.nio.file.FileVisitOption; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.tron.common.utils.FileUtil; - -@Slf4j(topic = "tool") -public class Util { - - /** - * Copy src to dest, if dest is a directory and already exists, throw Exception. - * - *

Note: This method is not rigorous, because all the dirs that its FileName - * is contained in List(subDirs) will be filtered, this may result in unpredictable result. - * just used in LiteFullNodeTool. - * - * @param src Path or File - * @param dest Path or File - * @param subDirs only the subDirs in {@code src} will be copied - * @throws IOException IOException - */ - public static void copyDatabases(Path src, Path dest, List subDirs) - throws IOException { - // create subdirs, as using parallel() to run, so should create dirs first. - subDirs.forEach(dir -> { - if (FileUtil.isExists(Paths.get(src.toString(), dir).toString())) { - try { - Files.walk(Paths.get(src.toString(), dir), FileVisitOption.FOLLOW_LINKS) - .forEach(source -> copy(source, dest.resolve(src.relativize(source)))); - } catch (IOException e) { - logger.error("copy database failed, src: {}, dest: {}, error: {}", - Paths.get(src.toString(), dir), Paths.get(dest.toString(), dir), e.getMessage()); - throw new RuntimeException(e); - } - } - }); - } - - public static void copy(Path source, Path dest) { - try { - // create hard link when file is .sst - if (source.toString().endsWith(".sst")) { - try { - Files.createLink(dest, source); - } catch (FileSystemException e) { - Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); - } - } else { - Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); - } - } catch (Exception e) { - throw new RuntimeException(e.getMessage(), e); - } - } -} diff --git a/framework/src/main/java/org/tron/tool/litefullnode/db/DBInterface.java b/framework/src/main/java/org/tron/tool/litefullnode/db/DBInterface.java deleted file mode 100644 index 8f30cbc5026..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/db/DBInterface.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.tron.tool.litefullnode.db; - -import java.io.Closeable; -import java.io.IOException; -import org.tron.tool.litefullnode.iterator.DBIterator; - -public interface DBInterface extends Closeable { - - byte[] get(byte[] key); - - void put(byte[] key, byte[] value); - - void delete(byte[] key); - - DBIterator iterator(); - - long size(); - - void close() throws IOException; - -} diff --git a/framework/src/main/java/org/tron/tool/litefullnode/db/LevelDBImpl.java b/framework/src/main/java/org/tron/tool/litefullnode/db/LevelDBImpl.java deleted file mode 100644 index 774c9a86146..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/db/LevelDBImpl.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.tron.tool.litefullnode.db; - -import com.google.common.collect.Streams; -import java.io.IOException; -import org.iq80.leveldb.DB; -import org.tron.tool.litefullnode.iterator.DBIterator; -import org.tron.tool.litefullnode.iterator.LevelDBIterator; - -public class LevelDBImpl implements DBInterface { - - private DB leveldb; - - public LevelDBImpl(DB leveldb) { - this.leveldb = leveldb; - } - - @Override - public byte[] get(byte[] key) { - return leveldb.get(key); - } - - @Override - public void put(byte[] key, byte[] value) { - leveldb.put(key, value); - } - - @Override - public void delete(byte[] key) { - leveldb.delete(key); - } - - @Override - public DBIterator iterator() { - return new LevelDBIterator(leveldb.iterator()); - } - - @Override - public long size() { - return Streams.stream(leveldb.iterator()).count(); - } - - @Override - public void close() throws IOException { - leveldb.close(); - } -} diff --git a/framework/src/main/java/org/tron/tool/litefullnode/db/RocksDBImpl.java b/framework/src/main/java/org/tron/tool/litefullnode/db/RocksDBImpl.java deleted file mode 100644 index d6dfc55f1bb..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/db/RocksDBImpl.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.tron.tool.litefullnode.db; - -import com.google.common.collect.Streams; -import java.io.IOException; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; -import org.tron.tool.litefullnode.iterator.DBIterator; -import org.tron.tool.litefullnode.iterator.RockDBIterator; - -public class RocksDBImpl implements DBInterface { - - private org.rocksdb.RocksDB rocksDB; - - public RocksDBImpl(org.rocksdb.RocksDB rocksDB) { - this.rocksDB = rocksDB; - } - - @Override - public byte[] get(byte[] key) { - try { - return rocksDB.get(key); - } catch (RocksDBException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public void put(byte[] key, byte[] value) { - try { - rocksDB.put(key, value); - } catch (RocksDBException e) { - e.printStackTrace(); - } - } - - @Override - public void delete(byte[] key) { - try { - rocksDB.delete(key); - } catch (RocksDBException e) { - e.printStackTrace(); - } - } - - @Override - public DBIterator iterator() { - return new RockDBIterator(rocksDB.newIterator()); - } - - @Override - public long size() { - RocksIterator iterator = rocksDB.newIterator(); - long size = 0; - for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - size++; - } - iterator.close(); - return size; - } - - @Override - public void close() throws IOException { - rocksDB.close(); - } -} diff --git a/framework/src/main/java/org/tron/tool/litefullnode/iterator/DBIterator.java b/framework/src/main/java/org/tron/tool/litefullnode/iterator/DBIterator.java deleted file mode 100644 index 363252e660e..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/iterator/DBIterator.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.tron.tool.litefullnode.iterator; - -import java.io.Closeable; - -public interface DBIterator extends Closeable { - - void seek(byte[] key); - - void seekToFirst(); - - boolean hasNext(); - - byte[] getKey(); - - byte[] getValue(); - - void next(); -} diff --git a/framework/src/main/java/org/tron/tool/litefullnode/iterator/LevelDBIterator.java b/framework/src/main/java/org/tron/tool/litefullnode/iterator/LevelDBIterator.java deleted file mode 100644 index d75b21ce67e..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/iterator/LevelDBIterator.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.tron.tool.litefullnode.iterator; - -import java.io.IOException; - -public class LevelDBIterator implements DBIterator { - - private org.iq80.leveldb.DBIterator iterator; - - public LevelDBIterator(org.iq80.leveldb.DBIterator iterator) { - this.iterator = iterator; - } - - @Override - public void seek(byte[] key) { - iterator.seek(key); - } - - @Override - public void seekToFirst() { - iterator.seekToFirst(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public byte[] getKey() { - return iterator.peekNext().getKey(); - } - - @Override - public byte[] getValue() { - return iterator.peekNext().getValue(); - } - - @Override - public void next() { - iterator.next(); - } - - @Override - public void close() throws IOException { - iterator.close(); - } -} diff --git a/framework/src/main/java/org/tron/tool/litefullnode/iterator/RockDBIterator.java b/framework/src/main/java/org/tron/tool/litefullnode/iterator/RockDBIterator.java deleted file mode 100644 index eb13330ebce..00000000000 --- a/framework/src/main/java/org/tron/tool/litefullnode/iterator/RockDBIterator.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.tron.tool.litefullnode.iterator; - -import java.io.IOException; -import org.rocksdb.RocksIterator; - -public class RockDBIterator implements DBIterator { - - private RocksIterator iterator; - - public RockDBIterator(RocksIterator iterator) { - this.iterator = iterator; - } - - @Override - public void seek(byte[] key) { - iterator.seek(key); - } - - @Override - public void seekToFirst() { - iterator.seekToFirst(); - } - - @Override - public boolean hasNext() { - return iterator.isValid(); - } - - @Override - public byte[] getKey() { - return iterator.key(); - } - - @Override - public byte[] getValue() { - return iterator.value(); - } - - @Override - public void next() { - iterator.next(); - } - - @Override - public void close() throws IOException { - iterator.close(); - } -} diff --git a/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java b/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java deleted file mode 100644 index d1da0bf00d8..00000000000 --- a/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java +++ /dev/null @@ -1,202 +0,0 @@ -package org.tron.program; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import java.io.File; -import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import org.junit.After; -import org.junit.Test; -import org.tron.api.WalletGrpc; -import org.tron.common.application.Application; -import org.tron.common.application.ApplicationFactory; -import org.tron.common.application.TronApplicationContext; -import org.tron.common.config.DbBackupConfig; -import org.tron.common.crypto.ECKey; -import org.tron.common.utils.FileUtil; -import org.tron.common.utils.PublicMethod; -import org.tron.common.utils.Utils; -import org.tron.core.config.DefaultConfig; -import org.tron.core.config.args.Args; -import org.tron.core.services.RpcApiService; -import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; -import org.tron.tool.litefullnode.LiteFullNodeTool; - -@Slf4j -@Deprecated -public class LiteFullNodeToolTest { - - private TronApplicationContext context; - private WalletGrpc.WalletBlockingStub blockingStubFull = null; - private ManagedChannel channelFull; - private Application appTest; - private String databaseDir; - - private static String dbPath = "output_lite_fn"; - - /** - * init logic. - */ - public void startApp() { - context = new TronApplicationContext(DefaultConfig.class); - appTest = ApplicationFactory.create(context); - appTest.addService(context.getBean(RpcApiService.class)); - appTest.addService(context.getBean(RpcApiServiceOnSolidity.class)); - appTest.startup(); - - String fullNode = String.format("%s:%d", "127.0.0.1", - Args.getInstance().getRpcPort()); - channelFull = ManagedChannelBuilder.forTarget(fullNode) - .usePlaintext() - .build(); - blockingStubFull = WalletGrpc.newBlockingStub(channelFull); - } - - /** - * Delete the database when exited. - */ - public static void destroy(String dbPath) { - File f = new File(dbPath); - if (f.exists()) { - if (FileUtil.deleteDir(f)) { - logger.info("Release resources successful."); - } else { - logger.info("Release resources failure."); - } - } - } - - /** - * shutdown the fullNode. - */ - public void shutdown() throws InterruptedException { - if (channelFull != null) { - channelFull.shutdown().awaitTermination(5, TimeUnit.SECONDS); - } - context.close(); - } - - public void init() { - destroy(dbPath); // delete if prev failed - Args.setParam(new String[]{"-d", dbPath, "-w"}, "config-localtest.conf"); - // allow account root - Args.getInstance().setAllowAccountStateRoot(1); - Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); - databaseDir = Args.getInstance().getStorage().getDbDirectory(); - // init dbBackupConfig to avoid NPE - Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance(); - } - - @After - public void clear() { - destroy(dbPath); - Args.clearParam(); - dbPath = "output_lite_fn"; - } - - @Test - public void testToolsWithLevelDB() throws InterruptedException { - logger.info("testToolsWithLevelDB start"); - testTools("LEVELDB", 1); - } - - @Test - public void testToolsWithLevelDBV2() throws InterruptedException { - logger.info("testToolsWithLevelDB start"); - testTools("LEVELDB", 2); - } - - @Test - public void testToolsWithRocksDB() throws InterruptedException { - logger.info("testToolsWithRocksDB start"); - testTools("ROCKSDB", 1); - } - - private void testTools(String dbType, int checkpointVersion) - throws InterruptedException { - dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis()); - init(); - final String[] argsForSnapshot = - new String[]{"-o", "split", "-t", "snapshot", "--fn-data-path", - dbPath + File.separator + databaseDir, "--dataset-path", - dbPath}; - final String[] argsForHistory = - new String[]{"-o", "split", "-t", "history", "--fn-data-path", - dbPath + File.separator + databaseDir, "--dataset-path", - dbPath}; - final String[] argsForMerge = - new String[]{"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, - "--dataset-path", dbPath + File.separator + "history"}; - Args.getInstance().getStorage().setDbEngine(dbType); - Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); - LiteFullNodeTool.setRecentBlks(3); - // start fullNode - startApp(); - // produce transactions for 18 seconds - generateSomeTransactions(18); - // stop the node - shutdown(); - // delete tran-cache - FileUtil.deleteDir(Paths.get(dbPath, databaseDir, "trans-cache").toFile()); - // generate snapshot - LiteFullNodeTool.main(argsForSnapshot); - // start fullNode - startApp(); - // produce transactions for 6 seconds - generateSomeTransactions(6); - // stop the node - shutdown(); - // generate history - LiteFullNodeTool.main(argsForHistory); - // backup original database to database_bak - File database = new File(Paths.get(dbPath, databaseDir).toString()); - if (!database.renameTo(new File(Paths.get(dbPath, databaseDir + "_bak").toString()))) { - throw new RuntimeException( - String.format("rename %s to %s failed", database.getPath(), - Paths.get(dbPath, databaseDir))); - } - // change snapshot to the new database - File snapshot = new File(Paths.get(dbPath, "snapshot").toString()); - if (!snapshot.renameTo(new File(Paths.get(dbPath, databaseDir).toString()))) { - throw new RuntimeException( - String.format("rename snapshot to %s failed", - Paths.get(dbPath, databaseDir))); - } - // start and validate the snapshot - startApp(); - generateSomeTransactions(6); - // stop the node - shutdown(); - // merge history - LiteFullNodeTool.main(argsForMerge); - // start and validate - startApp(); - generateSomeTransactions(6); - shutdown(); - LiteFullNodeTool.reSetRecentBlks(); - } - - private void generateSomeTransactions(int during) { - during *= 1000; // ms - int runTime = 0; - int sleepOnce = 100; - while (true) { - ECKey ecKey2 = new ECKey(Utils.getRandom()); - byte[] address = ecKey2.getAddress(); - - String sunPri = "cba92a516ea09f620a16ff7ee95ce0df1d56550a8babe9964981a7144c8a784a"; - byte[] sunAddress = PublicMethod.getFinalAddress(sunPri); - PublicMethod.sendcoin(address, 1L, - sunAddress, sunPri, blockingStubFull); - try { - Thread.sleep(sleepOnce); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if ((runTime += sleepOnce) > during) { - return; - } - } - } -} diff --git a/plugins/README.md b/plugins/README.md index 6807bfbb409..ed4235ea58e 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -66,11 +66,6 @@ DB copy provides a helper which can copy LevelDB or RocksDB data quickly on the java -jar Toolkit.jar db cp output-directory/database /tmp/databse ``` - -## DB Lite - -DB lite provides lite database, parameters are compatible with previous `LiteFullNodeTool`. - ### Available parameters: - `-o | --operate`: [split,merge], default: split. From c109b22fbc1530e5705cacf5d616ef8626161e53 Mon Sep 17 00:00:00 2001 From: liukai Date: Thu, 4 Jan 2024 14:54:47 +0800 Subject: [PATCH 23/61] feat(LiteFullNode): fix deleting titles --- plugins/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/README.md b/plugins/README.md index ed4235ea58e..19c952372c1 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -66,6 +66,10 @@ DB copy provides a helper which can copy LevelDB or RocksDB data quickly on the java -jar Toolkit.jar db cp output-directory/database /tmp/databse ``` +## DB Lite + +DB lite provides lite database, parameters are compatible with previous `LiteFullNodeTool`. + ### Available parameters: - `-o | --operate`: [split,merge], default: split. From db9f3beb942f7519d168d84745cfe861367ce35f Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Fri, 5 Jan 2024 14:25:25 +0800 Subject: [PATCH 24/61] feat(lite): optimize DbLite tool (#5647) --- .../main/java/org/tron/plugins/DbLite.java | 88 ++++--------------- 1 file changed, 15 insertions(+), 73 deletions(-) diff --git a/plugins/src/main/java/org/tron/plugins/DbLite.java b/plugins/src/main/java/org/tron/plugins/DbLite.java index ef7e73ec6d6..8804d6210b2 100644 --- a/plugins/src/main/java/org/tron/plugins/DbLite.java +++ b/plugins/src/main/java/org/tron/plugins/DbLite.java @@ -3,8 +3,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.primitives.Bytes; -import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import java.io.File; import java.io.FileNotFoundException; @@ -155,10 +153,10 @@ public void generateSnapshot(String sourceDir, String snapshotDir) { long start = System.currentTimeMillis(); snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString(); try { + mergeCheckpoint(sourceDir); hasEnoughBlock(sourceDir); List snapshotDbs = getSnapshotDbs(sourceDir); split(sourceDir, snapshotDir, snapshotDbs); - mergeCheckpoint2Snapshot(sourceDir, snapshotDir); // write genesisBlock , latest recent blocks and trans fillSnapshotBlockAndTransDb(sourceDir, snapshotDir); // save min block to info @@ -192,9 +190,9 @@ public void generateHistory(String sourceDir, String historyDir) { throw new IllegalStateException( String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir)); } + mergeCheckpoint(sourceDir); hasEnoughBlock(sourceDir); split(sourceDir, historyDir, archiveDbs); - mergeCheckpoint2History(sourceDir, historyDir); // save max block to info generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(), getLatestBlockHeaderNum(sourceDir)); @@ -263,15 +261,6 @@ private List getSnapshotDbs(String sourceDir) { return snapshotDbs; } - private void mergeCheckpoint2Snapshot(String sourceDir, String historyDir) { - List snapshotDbs = getSnapshotDbs(sourceDir); - mergeCheckpoint(sourceDir, historyDir, snapshotDbs); - } - - private void mergeCheckpoint2History(String sourceDir, String destDir) { - mergeCheckpoint(sourceDir, destDir, archiveDbs); - } - private void split(String sourceDir, String destDir, List dbs) throws IOException { logger.info("Begin to split the dbs."); spec.commandLine().getOut().println("Begin to split the dbs."); @@ -289,7 +278,7 @@ private void split(String sourceDir, String destDir, List dbs) throws IO FileUtils.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs); } - private void mergeCheckpoint(String sourceDir, String destDir, List destDbs) { + private void mergeCheckpoint(String sourceDir) { logger.info("Begin to merge checkpoint to dataset."); spec.commandLine().getOut().println("Begin to merge checkpoint to dataset."); try { @@ -298,18 +287,18 @@ private void mergeCheckpoint(String sourceDir, String destDir, List dest for (String cp : cpList) { DBInterface checkpointDb = DbTool.getDB( sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, cp); - recover(checkpointDb, destDir, destDbs); + recover(checkpointDb, sourceDir); } } else if (Paths.get(sourceDir, CHECKPOINT_DB).toFile().exists()) { DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); - recover(tmpDb, destDir, destDbs); + recover(tmpDb, sourceDir); } } catch (IOException | RocksDBException e) { throw new RuntimeException(e); } } - private void recover(DBInterface db, String destDir, List destDbs) + private void recover(DBInterface db, String destDir) throws IOException, RocksDBException { try (DBIterator iterator = db.iterator()) { for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { @@ -323,17 +312,15 @@ private void recover(DBInterface db, String destDir, List destDbs) byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length); byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); - if (destDbs != null && destDbs.contains(dbName)) { - DBInterface destDb = DbTool.getDB(destDir, dbName); - if (realValue != null) { - destDb.put(realKey, realValue); + DBInterface destDb = DbTool.getDB(destDir, dbName); + if (realValue != null) { + destDb.put(realKey, realValue); + } else { + byte op = value[0]; + if (DBUtils.Operator.DELETE.getValue() == op) { + destDb.delete(realKey); } else { - byte op = value[0]; - if (DBUtils.Operator.DELETE.getValue() == op) { - destDb.delete(realKey); - } else { - destDb.put(realKey, new byte[0]); - } + destDb.put(realKey, new byte[0]); } } } @@ -353,23 +340,7 @@ private void generateInfoProperties(String propertyfile, long num) } private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException { - // query latest_block_header_number from checkpoint first final String latestBlockHeaderNumber = "latest_block_header_number"; - List cpList = getCheckpointV2List(databaseDir); - DBInterface checkpointDb; - if (cpList.size() > 0) { - String lastestCp = cpList.get(cpList.size() - 1); - checkpointDb = DbTool.getDB( - databaseDir + "/" + DBUtils.CHECKPOINT_DB_V2, lastestCp); - } else { - checkpointDb = DbTool.getDB(databaseDir, CHECKPOINT_DB); - } - Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb, - latestBlockHeaderNumber.getBytes()); - if (blockNumber != null) { - return blockNumber; - } - // query from propertiesDb if checkpoint not contains latest_block_header_number DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME); return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber))) .map(ByteArray::toLong) @@ -377,14 +348,6 @@ private long getLatestBlockHeaderNum(String databaseDir) throws IOException, Roc () -> new IllegalArgumentException("not found latest block header number")); } - private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) { - byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key)); - if (value != null && value.length > 1) { - return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length)); - } - return null; - } - /** * recent blocks, trans and genesis block. */ @@ -451,15 +414,6 @@ private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDB return result; } - private static byte[] simpleEncode(String s) { - byte[] bytes = s.getBytes(); - byte[] length = Ints.toByteArray(bytes.length); - byte[] r = new byte[4 + bytes.length]; - System.arraycopy(length, 0, r, 0, 4); - System.arraycopy(bytes, 0, r, 4, bytes.length); - return r; - } - private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String liteDir) throws IOException, RocksDBException { logger.info("Check the compatibility of this history."); @@ -531,7 +485,6 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo) DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME); DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME); - ProgressBar.wrap(LongStream.rangeClosed(start, end) .boxed() .sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> { @@ -566,7 +519,6 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws return; } - Path bakDir = Paths.get(liteDir, BACKUP_DIR_PREFIX + START_TIME); logger.info("Begin to merge {} to database, start {} end {}.", bakDir, start, end); spec.commandLine().getOut() @@ -593,17 +545,7 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key) throws IOException, RocksDBException { - DBInterface sourceDb = DbTool.getDB(sourceDir, dbName); - DBInterface checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); - // get data from tmp first. - byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key)); - byte[] value; - if (isEmptyBytes(valueFromTmp)) { - value = sourceDb.get(key); - } else { - value = valueFromTmp.length == 1 - ? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length); - } + byte[] value = DbTool.getDB(sourceDir, dbName).get(key); if (isEmptyBytes(value)) { throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s", dbName, Arrays.toString(key))); From 52b33273b68441bf5b1bf3e97bdcb49ab1b2693c Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:21:25 +0800 Subject: [PATCH 25/61] feat(db): optimize old rewards withdrawal (#5406) --- .../org/tron/core/utils/ProposalUtil.java | 23 +- .../core/db/common/iterator/DBIterator.java | 70 ++++- .../db/common/iterator/RockStoreIterator.java | 66 ++++- .../db/common/iterator/StoreIterator.java | 63 +++- .../tron/core/service/MortgageService.java | 45 ++- .../tron/core/service/RewardViCalService.java | 273 +++++++++++++++++ .../core/store/DynamicPropertiesStore.java | 32 ++ .../org/tron/core/store/RewardViStore.java | 43 +++ .../common/parameter/CommonParameter.java | 4 + .../tron/common/prometheus/MetricKeys.java | 2 + .../common/prometheus/MetricsHistogram.java | 2 + .../src/main/java/org/tron/core/Constant.java | 1 + .../java/org/tron/core/config/Parameter.java | 5 +- .../src/main/java/org/tron/core/Wallet.java | 4 + .../java/org/tron/core/config/args/Args.java | 5 + .../tron/core/consensus/ProposalService.java | 4 + .../db/common/iterator/AbstractIterator.java | 41 --- .../common/iterator/AssetIssueIterator.java | 17 -- .../core/db/common/iterator/DBIterator.java | 9 - .../common/iterator/TransactionIterator.java | 22 -- .../db/common/iterator/WitnessIterator.java | 17 -- .../java/org/tron/core/db/DBIteratorTest.java | 110 ++++--- .../tron/core/services/ComputeRewardTest.java | 277 ++++++++++++++++++ .../core/services/ProposalServiceTest.java | 9 + 24 files changed, 969 insertions(+), 175 deletions(-) create mode 100644 chainbase/src/main/java/org/tron/core/service/RewardViCalService.java create mode 100755 chainbase/src/main/java/org/tron/core/store/RewardViStore.java delete mode 100644 framework/src/main/java/org/tron/core/db/common/iterator/AbstractIterator.java delete mode 100644 framework/src/main/java/org/tron/core/db/common/iterator/AssetIssueIterator.java delete mode 100755 framework/src/main/java/org/tron/core/db/common/iterator/DBIterator.java delete mode 100644 framework/src/main/java/org/tron/core/db/common/iterator/TransactionIterator.java delete mode 100644 framework/src/main/java/org/tron/core/db/common/iterator/WitnessIterator.java create mode 100644 framework/src/test/java/org/tron/core/services/ComputeRewardTest.java diff --git a/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java b/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java index 0f55bbae9b7..cf013266bd0 100644 --- a/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java +++ b/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java @@ -728,6 +728,26 @@ public static void validator(DynamicPropertiesStore dynamicPropertiesStore, } break; } + case ALLOW_OLD_REWARD_OPT: { + if (!forkController.pass(ForkBlockVersionEnum.VERSION_4_7_4)) { + throw new ContractValidateException( + "Bad chain parameter id [ALLOW_OLD_REWARD_OPT]"); + } + if (value != 1) { + throw new ContractValidateException( + "This value[ALLOW_OLD_REWARD_OPT] is only allowed to be 1"); + } + if (!dynamicPropertiesStore.useNewRewardAlgorithm()) { + throw new ContractValidateException( + "[ALLOW_NEW_REWARD] proposal must be approved " + + "before [ALLOW_OLD_REWARD_OPT] can be proposed"); + } + if (dynamicPropertiesStore.useNewRewardAlgorithmFromStart()) { + throw new ContractValidateException( + "no need old reward opt, ALLOW_NEW_REWARD from start cycle 1"); + } + break; + } default: break; } @@ -803,7 +823,8 @@ public enum ProposalType { // current value, value range DYNAMIC_ENERGY_MAX_FACTOR(75), // 0, [0, 100_000] ALLOW_TVM_SHANGHAI(76), // 0, 1 ALLOW_CANCEL_ALL_UNFREEZE_V2(77), // 0, 1 - MAX_DELEGATE_LOCK_PERIOD(78); // (86400, 10512000] + MAX_DELEGATE_LOCK_PERIOD(78), // (86400, 10512000] + ALLOW_OLD_REWARD_OPT(79); // 0, 1 private long code; diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java index f706623693f..ec8d7fd85be 100755 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java @@ -1,9 +1,77 @@ package org.tron.core.db.common.iterator; +import com.google.common.collect.Iterators; +import com.google.common.collect.UnmodifiableIterator; +import com.google.common.primitives.Bytes; import java.io.Closeable; import java.util.Iterator; import java.util.Map.Entry; +import java.util.NoSuchElementException; -public interface DBIterator extends Iterator>, Closeable { +public interface DBIterator extends Iterator>, AutoCloseable, Closeable { + void seek(byte[] key); + + void seekToFirst(); + + void seekToLast(); + + default UnmodifiableIterator> prefixQueryAfterThat + (byte[] key, byte[] afterThat) { + this.seek(afterThat == null ? key : afterThat); + return Iterators.filter(this, entry -> Bytes.indexOf(entry.getKey(), key) == 0); + } + + /** + * An iterator is either positioned at a key/value pair, or + * not valid. This method returns true iff the iterator is valid. + * + * REQUIRES: iterator not closed + * + * @throws IllegalStateException if the iterator is closed. + * @return an iterator is either positioned at a key/value pair + */ + boolean valid(); + + /** + * The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator. + * + * REQUIRES: valid() && !closed + * + * @throws IllegalStateException if the iterator is closed. + * @throws NoSuchElementException if the iterator is not valid. + * + * @return the key for the current entry + */ + byte[] getKey(); + + /** + * The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator. + * + * REQUIRES: valid() && !closed + * + * @throws IllegalStateException if the iterator is closed. + * @throws NoSuchElementException if the iterator is not valid. + * + * @return the value for the current entry + */ + byte[] getValue(); + + /** + * @throws IllegalStateException if the iterator is closed. + */ + void checkState(); + + /** + * @throws NoSuchElementException if the iterator is not valid. + */ + default void checkValid() { + if (!valid()) { + throw new NoSuchElementException(); + } + } } diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java index a8c4cff2066..1438b9247b5 100644 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.Map.Entry; import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksIterator; @@ -13,7 +14,7 @@ public final class RockStoreIterator implements DBIterator { private final RocksIterator dbIterator; private boolean first = true; - private boolean valid = true; + private final AtomicBoolean close = new AtomicBoolean(false); public RockStoreIterator(RocksIterator dbIterator) { this.dbIterator = dbIterator; @@ -21,12 +22,14 @@ public RockStoreIterator(RocksIterator dbIterator) { @Override public void close() throws IOException { - dbIterator.close(); + if (close.compareAndSet(false, true)) { + dbIterator.close(); + } } @Override public boolean hasNext() { - if (!valid) { + if (close.get()) { return false; } boolean hasNext = false; @@ -37,13 +40,12 @@ public boolean hasNext() { first = false; } if (!(hasNext = dbIterator.isValid())) { // false is last item - dbIterator.close(); - valid = false; + close(); } } catch (Exception e) { logger.error(e.getMessage(), e); try { - dbIterator.close(); + close(); } catch (Exception e1) { logger.error(e.getMessage(), e); } @@ -53,7 +55,7 @@ public boolean hasNext() { @Override public Entry next() { - if (!valid) { + if (close.get()) { throw new NoSuchElementException(); } byte[] key = dbIterator.key(); @@ -76,4 +78,52 @@ public byte[] setValue(byte[] value) { } }; } -} \ No newline at end of file + + @Override + public void seek(byte[] key) { + checkState(); + dbIterator.seek(key); + this.first = false; + } + + @Override + public void seekToFirst() { + checkState(); + dbIterator.seekToFirst(); + this.first = false; + } + + @Override + public void seekToLast() { + checkState(); + dbIterator.seekToLast(); + this.first = false; + } + + @Override + public boolean valid() { + checkState(); + return dbIterator.isValid(); + } + + @Override + public byte[] getKey() { + checkState(); + checkValid(); + return dbIterator.key(); + } + + @Override + public byte[] getValue() { + checkState(); + checkValid(); + return dbIterator.value(); + } + + @Override + public void checkState() { + if (close.get()) { + throw new IllegalStateException("iterator has been closed"); + } + } +} diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java index 292bb421e54..4c635660ea6 100755 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.Map.Entry; import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.DBIterator; @@ -13,7 +14,7 @@ public final class StoreIterator implements org.tron.core.db.common.iterator.DBI private final DBIterator dbIterator; private boolean first = true; - private boolean valid = true; + private final AtomicBoolean close = new AtomicBoolean(false); public StoreIterator(DBIterator dbIterator) { this.dbIterator = dbIterator; @@ -21,12 +22,14 @@ public StoreIterator(DBIterator dbIterator) { @Override public void close() throws IOException { - dbIterator.close(); + if (close.compareAndSet(false, true)) { + dbIterator.close(); + } } @Override public boolean hasNext() { - if (!valid) { + if (close.get()) { return false; } @@ -39,8 +42,7 @@ public boolean hasNext() { } if (!(hasNext = dbIterator.hasNext())) { // false is last item - dbIterator.close(); - valid = false; + close(); } } catch (Exception e) { logger.error(e.getMessage(), e); @@ -51,7 +53,7 @@ public boolean hasNext() { @Override public Entry next() { - if (!valid) { + if (close.get()) { throw new NoSuchElementException(); } return dbIterator.next(); @@ -61,4 +63,53 @@ public Entry next() { public void remove() { throw new UnsupportedOperationException(); } + + @Override + public void seek(byte[] key) { + checkState(); + dbIterator.seek(key); + this.first = false; + } + + @Override + public void seekToFirst() { + checkState(); + dbIterator.seekToFirst(); + this.first = false; + } + + @Override + public void seekToLast() { + checkState(); + dbIterator.seekToLast(); + this.first = false; + } + + @Override + public boolean valid() { + checkState(); + return dbIterator.hasNext(); + } + + @Override + public byte[] getKey() { + checkState(); + checkValid(); + return dbIterator.peekNext().getKey(); + } + + @Override + public byte[] getValue() { + checkState(); + checkValid(); + return dbIterator.peekNext().getValue(); + } + + @Override + public void checkState() { + if (close.get()) { + throw new IllegalStateException("iterator has been closed"); + } + } } + diff --git a/chainbase/src/main/java/org/tron/core/service/MortgageService.java b/chainbase/src/main/java/org/tron/core/service/MortgageService.java index e9b00a38201..805245d53f2 100644 --- a/chainbase/src/main/java/org/tron/core/service/MortgageService.java +++ b/chainbase/src/main/java/org/tron/core/service/MortgageService.java @@ -4,12 +4,15 @@ import java.math.BigInteger; import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.bouncycastle.util.encoders.Hex; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.utils.Pair; import org.tron.common.utils.StringUtil; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.WitnessCapsule; @@ -18,7 +21,6 @@ import org.tron.core.store.DelegationStore; import org.tron.core.store.DynamicPropertiesStore; import org.tron.core.store.WitnessStore; -import org.tron.protos.Protocol.Vote; @Slf4j(topic = "mortgage") @Component @@ -37,6 +39,9 @@ public class MortgageService { @Setter private AccountStore accountStore; + @Autowired + private RewardViCalService rewardViCalService; + public void initStore(WitnessStore witnessStore, DelegationStore delegationStore, DynamicPropertiesStore dynamicPropertiesStore, AccountStore accountStore) { this.witnessStore = witnessStore; @@ -162,21 +167,21 @@ public long queryReward(byte[] address) { return reward + accountCapsule.getAllowance(); } - private long computeReward(long cycle, AccountCapsule accountCapsule) { + private long computeReward(long cycle, List> votes) { long reward = 0; - for (Vote vote : accountCapsule.getVotesList()) { - byte[] srAddress = vote.getVoteAddress().toByteArray(); + for (Pair vote : votes) { + byte[] srAddress = vote.getKey(); long totalReward = delegationStore.getReward(cycle, srAddress); + if (totalReward <= 0) { + continue; + } long totalVote = delegationStore.getWitnessVote(cycle, srAddress); if (totalVote == DelegationStore.REMARK || totalVote == 0) { continue; } - long userVote = vote.getVoteCount(); + long userVote = vote.getValue(); double voteRate = (double) userVote / totalVote; reward += voteRate * totalReward; - logger.debug("ComputeReward {}, {}, {}, {}, {}, {}, {}.", cycle, - Hex.toHexString(accountCapsule.getAddress().toByteArray()), Hex.toHexString(srAddress), - userVote, totalVote, totalReward, reward); } return reward; } @@ -197,23 +202,24 @@ private long computeReward(long beginCycle, long endCycle, AccountCapsule accoun long reward = 0; long newAlgorithmCycle = dynamicPropertiesStore.getNewRewardAlgorithmEffectiveCycle(); + List> srAddresses = accountCapsule.getVotesList().stream() + .map(vote -> new Pair<>(vote.getVoteAddress().toByteArray(), vote.getVoteCount())) + .collect(Collectors.toList()); if (beginCycle < newAlgorithmCycle) { long oldEndCycle = Math.min(endCycle, newAlgorithmCycle); - for (long cycle = beginCycle; cycle < oldEndCycle; cycle++) { - reward += computeReward(cycle, accountCapsule); - } + reward = getOldReward(beginCycle, oldEndCycle, srAddresses); beginCycle = oldEndCycle; } if (beginCycle < endCycle) { - for (Vote vote : accountCapsule.getVotesList()) { - byte[] srAddress = vote.getVoteAddress().toByteArray(); + for (Pair vote : srAddresses) { + byte[] srAddress = vote.getKey(); BigInteger beginVi = delegationStore.getWitnessVi(beginCycle - 1, srAddress); BigInteger endVi = delegationStore.getWitnessVi(endCycle - 1, srAddress); BigInteger deltaVi = endVi.subtract(beginVi); if (deltaVi.signum() <= 0) { continue; } - long userVote = vote.getVoteCount(); + long userVote = vote.getValue(); reward += deltaVi.multiply(BigInteger.valueOf(userVote)) .divide(DelegationStore.DECIMAL_OF_VI_REWARD).longValue(); } @@ -257,4 +263,15 @@ private void sortWitness(List list) { list.sort(Comparator.comparingLong((ByteString b) -> getWitnessByAddress(b).getVoteCount()) .reversed().thenComparing(Comparator.comparingInt(ByteString::hashCode).reversed())); } + + private long getOldReward(long begin, long end, List> votes) { + if (dynamicPropertiesStore.allowOldRewardOpt()) { + return rewardViCalService.getNewRewardAlgorithmReward(begin, end, votes); + } + long reward = 0; + for (long cycle = begin; cycle < end; cycle++) { + reward += computeReward(cycle, votes); + } + return reward; + } } diff --git a/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java b/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java new file mode 100644 index 00000000000..acb16142b3f --- /dev/null +++ b/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java @@ -0,0 +1,273 @@ +package org.tron.core.service; + +import static org.tron.core.store.DelegationStore.DECIMAL_OF_VI_REWARD; +import static org.tron.core.store.DelegationStore.REMARK; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Streams; +import com.google.common.primitives.Bytes; +import com.google.protobuf.ByteString; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.LongStream; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.bouncycastle.util.encoders.Hex; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.tron.common.error.TronDBException; +import org.tron.common.es.ExecutorServiceManager; +import org.tron.common.parameter.CommonParameter; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Pair; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.capsule.utils.MerkleTree; +import org.tron.core.db.common.iterator.DBIterator; +import org.tron.core.db2.common.DB; +import org.tron.core.store.DelegationStore; +import org.tron.core.store.DynamicPropertiesStore; +import org.tron.core.store.RewardViStore; +import org.tron.core.store.WitnessStore; + +@Component +@Slf4j(topic = "rewardViCalService") +public class RewardViCalService { + + private final DB propertiesStore; + private final DB delegationStore; + private final DB witnessStore; + + @Autowired + private RewardViStore rewardViStore; + + private static final byte[] IS_DONE_KEY = new byte[]{0x00}; + private static final byte[] IS_DONE_VALUE = new byte[]{0x01}; + + private long newRewardCalStartCycle = Long.MAX_VALUE; + + private volatile long lastBlockNumber = -1; + + @VisibleForTesting + @Setter + private Sha256Hash rewardViRoot = Sha256Hash.wrap( + ByteString.fromHex("9debcb9924055500aaae98cdee10501c5c39d4daa75800a996f4bdda73dbccd8")); + + private final CountDownLatch lock = new CountDownLatch(1); + + private final ScheduledExecutorService es = ExecutorServiceManager + .newSingleThreadScheduledExecutor("rewardViCalService"); + + + @Autowired + public RewardViCalService(@Autowired DynamicPropertiesStore propertiesStore, + @Autowired DelegationStore delegationStore, @Autowired WitnessStore witnessStore) { + this.propertiesStore = propertiesStore.getDb(); + this.delegationStore = delegationStore.getDb(); + this.witnessStore = witnessStore.getDb(); + } + + @PostConstruct + private void init() { + es.scheduleWithFixedDelay(this::maybeRun, 0, 3, TimeUnit.SECONDS); + } + + private boolean enableNewRewardAlgorithm() { + this.newRewardCalStartCycle = this.getNewRewardAlgorithmEffectiveCycle(); + boolean ret = this.newRewardCalStartCycle != Long.MAX_VALUE; + if (ret && lastBlockNumber == -1) { + lastBlockNumber = this.getLatestBlockHeaderNumber(); + } + return ret; + } + + private boolean isDone() { + return rewardViStore.has(IS_DONE_KEY); + } + + private void maybeRun() { + if (enableNewRewardAlgorithm()) { + if (this.newRewardCalStartCycle > 1) { + if (isDone()) { + this.clearUp(true); + logger.info("rewardViCalService is already done"); + } else { + if (this.getLatestBlockHeaderNumber() > lastBlockNumber) { + // checkpoint is flushed to db, so we can start rewardViCalService + startRewardCal(); + clearUp(true); + } else { + logger.info("startRewardCal will run after checkpoint is flushed to db"); + } + } + } else { + clearUp(false); + logger.info("rewardViCalService is no need to run"); + } + } + } + + private void clearUp(boolean isDone) { + lock.countDown(); + if (isDone) { + calcMerkleRoot(); + } + es.shutdown(); + } + + @PreDestroy + private void destroy() { + es.shutdownNow(); + } + + + public long getNewRewardAlgorithmReward(long beginCycle, long endCycle, + List> votes) { + if (!rewardViStore.has(IS_DONE_KEY)) { + logger.warn("rewardViCalService is not done, wait for it"); + try { + lock.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TronDBException(e); + } + } + + long reward = 0; + if (beginCycle < endCycle) { + for (Pair vote : votes) { + byte[] srAddress = vote.getKey(); + BigInteger beginVi = getWitnessVi(beginCycle - 1, srAddress); + BigInteger endVi = getWitnessVi(endCycle - 1, srAddress); + BigInteger deltaVi = endVi.subtract(beginVi); + if (deltaVi.signum() <= 0) { + continue; + } + long userVote = vote.getValue(); + reward += deltaVi.multiply(BigInteger.valueOf(userVote)) + .divide(DelegationStore.DECIMAL_OF_VI_REWARD).longValue(); + } + } + return reward; + + } + + private void calcMerkleRoot() { + logger.info("calcMerkleRoot start"); + DBIterator iterator = rewardViStore.iterator(); + iterator.seekToFirst(); + ArrayList ids = Streams.stream(iterator) + .map(this::getHash) + .collect(Collectors.toCollection(ArrayList::new)); + + Sha256Hash rewardViRootLocal = MerkleTree.getInstance().createTree(ids).getRoot().getHash(); + if (!Objects.equals(rewardViRoot, rewardViRootLocal)) { + logger.error("merkle root mismatch, expect: {}, actual: {}", + rewardViRoot, rewardViRootLocal); + } + logger.info("calcMerkleRoot: {}", rewardViRootLocal); + } + + private Sha256Hash getHash(Map.Entry entry) { + return Sha256Hash.of(CommonParameter.getInstance().isECKeyCryptoEngine(), + Bytes.concat(entry.getKey(), entry.getValue())); + } + + private void startRewardCal() { + logger.info("rewardViCalService start"); + rewardViStore.reset(); + DBIterator iterator = (DBIterator) witnessStore.iterator(); + iterator.seekToFirst(); + iterator.forEachRemaining(e -> accumulateWitnessReward(e.getKey())); + rewardViStore.put(IS_DONE_KEY, IS_DONE_VALUE); + logger.info("rewardViCalService is done"); + + } + + private void accumulateWitnessReward(byte[] witness) { + long startCycle = 1; + LongStream.range(startCycle, newRewardCalStartCycle) + .forEach(cycle -> accumulateWitnessVi(cycle, witness)); + } + + private void accumulateWitnessVi(long cycle, byte[] address) { + BigInteger preVi = getWitnessVi(cycle - 1, address); + long voteCount = getWitnessVote(cycle, address); + long reward = getReward(cycle, address); + if (reward == 0 || voteCount == 0) { // Just forward pre vi + if (!BigInteger.ZERO.equals(preVi)) { // Zero vi will not be record + setWitnessVi(cycle, address, preVi); + } + } else { // Accumulate delta vi + BigInteger deltaVi = BigInteger.valueOf(reward) + .multiply(DECIMAL_OF_VI_REWARD) + .divide(BigInteger.valueOf(voteCount)); + setWitnessVi(cycle, address, preVi.add(deltaVi)); + } + } + + private void setWitnessVi(long cycle, byte[] address, BigInteger value) { + byte[] k = buildViKey(cycle, address); + byte[] v = value.toByteArray(); + rewardViStore.put(k, v); + } + + private BigInteger getWitnessVi(long cycle, byte[] address) { + + byte[] v = rewardViStore.get(buildViKey(cycle, address)); + if (v == null) { + return BigInteger.ZERO; + } else { + return new BigInteger(v); + } + } + + private byte[] buildViKey(long cycle, byte[] address) { + return generateKey(cycle, address, "vi"); + } + + private long getReward(long cycle, byte[] address) { + byte[] value = this.delegationStore.get(generateKey(cycle, address, "reward")); + return value == null ? 0 : ByteArray.toLong(value); + } + + private long getWitnessVote(long cycle, byte[] address) { + byte[] value = this.delegationStore.get(generateKey(cycle, address, "vote")); + return value == null ? REMARK : ByteArray.toLong(value); + } + + private byte[] generateKey(long cycle, byte[] address, String suffix) { + return generateKey(cycle + "", address, suffix); + } + + private byte[] generateKey(String prefix, byte[] address, String suffix) { + StringBuilder sb = new StringBuilder(); + if (prefix != null) { + sb.append(prefix).append("-"); + } + sb.append(Hex.toHexString(address)); + if (suffix != null) { + sb.append("-").append(suffix); + } + return sb.toString().getBytes(); + } + + private long getNewRewardAlgorithmEffectiveCycle() { + byte[] value = this.propertiesStore.get("NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE".getBytes()); + return value == null ? Long.MAX_VALUE : ByteArray.toLong(value); + } + + private long getLatestBlockHeaderNumber() { + byte[] value = this.propertiesStore.get("latest_block_header_number".getBytes()); + return value == null ? 1 : ByteArray.toLong(value); + } +} + diff --git a/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java b/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java index f3059d31558..23693f9bab6 100644 --- a/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java +++ b/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java @@ -217,6 +217,8 @@ public class DynamicPropertiesStore extends TronStoreWithRevoking private static final byte[] MAX_DELEGATE_LOCK_PERIOD = "MAX_DELEGATE_LOCK_PERIOD".getBytes(); + private static final byte[] ALLOW_OLD_REWARD_OPT = "ALLOW_OLD_REWARD_OPT".getBytes(); + @Autowired private DynamicPropertiesStore(@Value("properties") String dbName) { super(dbName); @@ -2523,6 +2525,10 @@ public boolean useNewRewardAlgorithm() { return getNewRewardAlgorithmEffectiveCycle() != Long.MAX_VALUE; } + public boolean useNewRewardAlgorithmFromStart() { + return getNewRewardAlgorithmEffectiveCycle() == 1; + } + public void saveNewRewardAlgorithmEffectiveCycle() { if (getNewRewardAlgorithmEffectiveCycle() == Long.MAX_VALUE) { long currentCycle = getCurrentCycleNumber(); @@ -2833,6 +2839,32 @@ public boolean supportMaxDelegateLockPeriod() { getUnfreezeDelayDays() > 0; } + /** + * @require NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE != Long.MAX_VALUE + * @require NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE > 1 + */ + public void saveAllowOldRewardOpt(long allowOldRewardOpt) { + if (useNewRewardAlgorithm()) { + if (useNewRewardAlgorithmFromStart()) { + throw new IllegalStateException("no need old reward opt, ALLOW_NEW_REWARD from start"); + } + this.put(ALLOW_OLD_REWARD_OPT, new BytesCapsule(ByteArray.fromLong(allowOldRewardOpt))); + } else { + throw new IllegalStateException("not support old reward opt, ALLOW_NEW_REWARD not set"); + } + } + + public boolean allowOldRewardOpt() { + return getAllowOldRewardOpt() == 1L; + } + + public long getAllowOldRewardOpt() { + return Optional.ofNullable(getUnchecked(ALLOW_OLD_REWARD_OPT)) + .map(BytesCapsule::getData) + .map(ByteArray::toLong) + .orElse(CommonParameter.getInstance().getAllowOldRewardOpt()); + } + private static class DynamicResourceProperties { private static final byte[] ONE_DAY_NET_LIMIT = "ONE_DAY_NET_LIMIT".getBytes(); diff --git a/chainbase/src/main/java/org/tron/core/store/RewardViStore.java b/chainbase/src/main/java/org/tron/core/store/RewardViStore.java new file mode 100755 index 00000000000..f173cecc00f --- /dev/null +++ b/chainbase/src/main/java/org/tron/core/store/RewardViStore.java @@ -0,0 +1,43 @@ +package org.tron.core.store; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.tron.core.db.TronDatabase; +import org.tron.core.db.common.iterator.DBIterator; + +@Slf4j(topic = "DB") +@Component +public class RewardViStore extends TronDatabase { + + @Autowired + private RewardViStore(@Value("reward-vi") String dbName) { + super(dbName); + } + + @Override + public byte[] get(byte[] key) { + return dbSource.getData(key); + } + + @Override + public void put(byte[] key, byte[] item) { + dbSource.putData(key, item); + } + + @Override + public void delete(byte[] key) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean has(byte[] key) { + return dbSource.getData(key) != null; + } + + @Override + public DBIterator iterator() { + return ((DBIterator) dbSource.iterator()); + } +} diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index b75eb682932..95a1eb2d0ae 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -662,6 +662,10 @@ public class CommonParameter { @Setter public int maxUnsolidifiedBlocks; + @Getter + @Setter + public long allowOldRewardOpt; + private static double calcMaxTimeRatio() { //return max(2.0, min(5.0, 5 * 4.0 / max(Runtime.getRuntime().availableProcessors(), 1))); return 5.0; diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 87ab6fae0a3..7e85771000e 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -62,6 +62,8 @@ public static class Histogram { public static final String MESSAGE_PROCESS_LATENCY = "tron:message_process_latency_seconds"; public static final String BLOCK_FETCH_LATENCY = "tron:block_fetch_latency_seconds"; public static final String BLOCK_RECEIVE_DELAY = "tron:block_receive_delay_seconds"; + public static final String DO_REWARD_CAL_DELAY = "tron:do_reward_cal_seconds"; + private Histogram() { throw new IllegalStateException("Histogram"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java b/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java index 556db10feb5..d776474c92e 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java @@ -48,6 +48,8 @@ public class MetricsHistogram { init(MetricKeys.Histogram.BLOCK_FETCH_LATENCY, "fetch block latency."); init(MetricKeys.Histogram.BLOCK_RECEIVE_DELAY, "receive block delay time, receiveTime - blockTime."); + init(MetricKeys.Histogram.DO_REWARD_CAL_DELAY, + "do reward cal delay time.", "depth"); } private MetricsHistogram() { diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index 634d08b79a9..2cd9ea95f15 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -375,4 +375,5 @@ public class Constant { public static final String UNSOLIDIFIED_BLOCK_CHECK = "node.unsolidifiedBlockCheck"; public static final String MAX_UNSOLIDIFIED_BLOCKS = "node.maxUnsolidifiedBlocks"; + public static final String COMMITTEE_ALLOW_OLD_REWARD_OPT = "committee.allowOldRewardOpt"; } diff --git a/common/src/main/java/org/tron/core/config/Parameter.java b/common/src/main/java/org/tron/core/config/Parameter.java index b1a948e9fdf..247826af77a 100644 --- a/common/src/main/java/org/tron/core/config/Parameter.java +++ b/common/src/main/java/org/tron/core/config/Parameter.java @@ -22,7 +22,8 @@ public enum ForkBlockVersionEnum { VERSION_4_6(25, 1596780000000L, 80), VERSION_4_7(26, 1596780000000L, 80), VERSION_4_7_1(27, 1596780000000L, 80), - VERSION_4_7_2(28, 1596780000000L, 80); + VERSION_4_7_2(28, 1596780000000L, 80), + VERSION_4_7_4(29, 1596780000000L, 80); // if add a version, modify BLOCK_VERSION simultaneously @Getter @@ -71,7 +72,7 @@ public class ChainConstant { public static final int SINGLE_REPEAT = 1; public static final int BLOCK_FILLED_SLOTS_NUMBER = 128; public static final int MAX_FROZEN_NUMBER = 1; - public static final int BLOCK_VERSION = 28; + public static final int BLOCK_VERSION = 29; public static final long FROZEN_PERIOD = 86_400_000L; public static final long DELEGATE_PERIOD = 3 * 86_400_000L; public static final long TRX_PRECISION = 1000_000L; diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index b14fcdb1167..cd2e26aa6f7 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -1329,6 +1329,10 @@ public Protocol.ChainParameters getChainParameters() { .setKey("getMaxDelegateLockPeriod") .setValue(dbManager.getDynamicPropertiesStore().getMaxDelegateLockPeriod()) .build()); + builder.addChainParameter(Protocol.ChainParameters.ChainParameter.newBuilder() + .setKey("getAllowOldRewardOpt") + .setValue(dbManager.getDynamicPropertiesStore().getAllowOldRewardOpt()) + .build()); return builder.build(); } diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 4321ad464bc..9694668af01 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -231,6 +231,7 @@ public static void clearParam() { PARAMETER.allowTvmShangHai = 0; PARAMETER.unsolidifiedBlockCheck = true; PARAMETER.maxUnsolidifiedBlocks = 1000; + PARAMETER.allowOldRewardOpt = 0; } /** @@ -1191,6 +1192,10 @@ public static void setParam(final String[] args, final String confFileName) { config.hasPath(Constant.MAX_UNSOLIDIFIED_BLOCKS) ? config .getInt(Constant.MAX_UNSOLIDIFIED_BLOCKS) : 1000; + PARAMETER.allowOldRewardOpt = + config.hasPath(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) ? config + .getInt(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) : 0; + logConfig(); } diff --git a/framework/src/main/java/org/tron/core/consensus/ProposalService.java b/framework/src/main/java/org/tron/core/consensus/ProposalService.java index 2cbf0c053d6..58da117cfc6 100644 --- a/framework/src/main/java/org/tron/core/consensus/ProposalService.java +++ b/framework/src/main/java/org/tron/core/consensus/ProposalService.java @@ -355,6 +355,10 @@ public static boolean process(Manager manager, ProposalCapsule proposalCapsule) manager.getDynamicPropertiesStore().saveMaxDelegateLockPeriod(entry.getValue()); break; } + case ALLOW_OLD_REWARD_OPT: { + manager.getDynamicPropertiesStore().saveAllowOldRewardOpt(entry.getValue()); + break; + } default: find = false; break; diff --git a/framework/src/main/java/org/tron/core/db/common/iterator/AbstractIterator.java b/framework/src/main/java/org/tron/core/db/common/iterator/AbstractIterator.java deleted file mode 100644 index ca3801619a8..00000000000 --- a/framework/src/main/java/org/tron/core/db/common/iterator/AbstractIterator.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.tron.core.db.common.iterator; - -import com.google.common.collect.Maps; -import com.google.common.reflect.TypeToken; -import java.lang.reflect.InvocationTargetException; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; - -public abstract class AbstractIterator implements Iterator> { - - protected Iterator> iterator; - private TypeToken typeToken = new TypeToken(getClass()) { - }; - - public AbstractIterator(Iterator> iterator) { - this.iterator = iterator; - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - protected T of(byte[] value) { - try { - @SuppressWarnings("unchecked") - T t = (T) typeToken.getRawType().getConstructor(byte[].class).newInstance(value); - return t; - } catch (InstantiationException | NoSuchMethodException - | InvocationTargetException | IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - @Override - public Map.Entry next() { - Entry entry = iterator.next(); - return Maps.immutableEntry(entry.getKey(), of(entry.getValue())); - } -} diff --git a/framework/src/main/java/org/tron/core/db/common/iterator/AssetIssueIterator.java b/framework/src/main/java/org/tron/core/db/common/iterator/AssetIssueIterator.java deleted file mode 100644 index e52e5a93b46..00000000000 --- a/framework/src/main/java/org/tron/core/db/common/iterator/AssetIssueIterator.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.tron.core.db.common.iterator; - -import java.util.Iterator; -import java.util.Map.Entry; -import org.tron.core.capsule.AssetIssueCapsule; - -public class AssetIssueIterator extends AbstractIterator { - - public AssetIssueIterator(Iterator> iterator) { - super(iterator); - } - - @Override - protected AssetIssueCapsule of(byte[] value) { - return new AssetIssueCapsule(value); - } -} \ No newline at end of file diff --git a/framework/src/main/java/org/tron/core/db/common/iterator/DBIterator.java b/framework/src/main/java/org/tron/core/db/common/iterator/DBIterator.java deleted file mode 100755 index f706623693f..00000000000 --- a/framework/src/main/java/org/tron/core/db/common/iterator/DBIterator.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.tron.core.db.common.iterator; - -import java.io.Closeable; -import java.util.Iterator; -import java.util.Map.Entry; - -public interface DBIterator extends Iterator>, Closeable { - -} diff --git a/framework/src/main/java/org/tron/core/db/common/iterator/TransactionIterator.java b/framework/src/main/java/org/tron/core/db/common/iterator/TransactionIterator.java deleted file mode 100644 index eb61f38b4b3..00000000000 --- a/framework/src/main/java/org/tron/core/db/common/iterator/TransactionIterator.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.tron.core.db.common.iterator; - -import java.util.Iterator; -import java.util.Map.Entry; -import org.tron.core.capsule.TransactionCapsule; -import org.tron.core.exception.BadItemException; - -public class TransactionIterator extends AbstractIterator { - - public TransactionIterator(Iterator> iterator) { - super(iterator); - } - - @Override - protected TransactionCapsule of(byte[] value) { - try { - return new TransactionCapsule(value); - } catch (BadItemException e) { - throw new RuntimeException(e); - } - } -} diff --git a/framework/src/main/java/org/tron/core/db/common/iterator/WitnessIterator.java b/framework/src/main/java/org/tron/core/db/common/iterator/WitnessIterator.java deleted file mode 100644 index 06837345141..00000000000 --- a/framework/src/main/java/org/tron/core/db/common/iterator/WitnessIterator.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.tron.core.db.common.iterator; - -import java.util.Iterator; -import java.util.Map.Entry; -import org.tron.core.capsule.WitnessCapsule; - -public class WitnessIterator extends AbstractIterator { - - public WitnessIterator(Iterator> iterator) { - super(iterator); - } - - @Override - protected WitnessCapsule of(byte[] value) { - return new WitnessCapsule(value); - } -} diff --git a/framework/src/test/java/org/tron/core/db/DBIteratorTest.java b/framework/src/test/java/org/tron/core/db/DBIteratorTest.java index a55d1b04d4d..b4f7ca424c0 100644 --- a/framework/src/test/java/org/tron/core/db/DBIteratorTest.java +++ b/framework/src/test/java/org/tron/core/db/DBIteratorTest.java @@ -5,86 +5,122 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.NoSuchElementException; import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -import org.tron.common.utils.FileUtil; import org.tron.core.db.common.iterator.RockStoreIterator; import org.tron.core.db.common.iterator.StoreIterator; public class DBIteratorTest { - @BeforeClass - public static void init() { - File file = Paths.get("database-iterator").toFile(); - if (!file.exists()) { - file.mkdirs(); - } - } + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); - @AfterClass - public static void clear() { - File file = Paths.get("database-iterator").toFile(); - if (file.exists()) { - FileUtil.deleteDir(Paths.get("database-iterator").toFile()); - } - } + @Rule + public final ExpectedException thrown = ExpectedException.none(); @Test public void testLevelDb() throws IOException { - File file = new File("database-iterator/testLevelDb"); - try { - DB db = factory.open(file, new Options().createIfMissing(true)); + File file = temporaryFolder.newFolder(); + try (DB db = factory.open(file, new Options().createIfMissing(true))) { db.put("1".getBytes(StandardCharsets.UTF_8), "1".getBytes(StandardCharsets.UTF_8)); db.put("2".getBytes(StandardCharsets.UTF_8), "2".getBytes(StandardCharsets.UTF_8)); - StoreIterator iterator = new StoreIterator(db.iterator()); + StoreIterator iterator = new StoreIterator(db.iterator()); + iterator.seekToFirst(); + Assert.assertArrayEquals("1".getBytes(StandardCharsets.UTF_8), iterator.getKey()); + Assert.assertArrayEquals("1".getBytes(StandardCharsets.UTF_8), iterator.next().getValue()); + Assert.assertTrue(iterator.hasNext()); + + Assert.assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), iterator.getValue()); + Assert.assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), iterator.next().getKey()); + Assert.assertFalse(iterator.hasNext()); + + try { + iterator.seekToLast(); + } catch (Exception e) { + Assert.assertTrue(e instanceof IllegalStateException); + } + + iterator = new StoreIterator(db.iterator()); + iterator.seekToLast(); + Assert.assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), iterator.getKey()); + Assert.assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), iterator.getValue()); + iterator.seekToFirst(); while (iterator.hasNext()) { iterator.next(); } Assert.assertFalse(iterator.hasNext()); try { - iterator.next(); + iterator.getKey(); } catch (Exception e) { - Assert.assertTrue(e instanceof NoSuchElementException); + Assert.assertTrue(e instanceof IllegalStateException); } - db.close(); - } finally { - factory.destroy(file, new Options()); + try { + iterator.getValue(); + } catch (Exception e) { + Assert.assertTrue(e instanceof IllegalStateException); + } + thrown.expect(NoSuchElementException.class); + iterator.next(); } } @Test - public void testRocksDb() throws RocksDBException { - File file = new File("database-iterator/testRocksDb"); - try (org.rocksdb.Options options = new org.rocksdb.Options().setCreateIfMissing(true)) { - RocksDB db = RocksDB.open(options, file.toString()); + public void testRocksDb() throws RocksDBException, IOException { + File file = temporaryFolder.newFolder(); + try (org.rocksdb.Options options = new org.rocksdb.Options().setCreateIfMissing(true); + RocksDB db = RocksDB.open(options, file.toString())) { db.put("1".getBytes(StandardCharsets.UTF_8), "1".getBytes(StandardCharsets.UTF_8)); db.put("2".getBytes(StandardCharsets.UTF_8), "2".getBytes(StandardCharsets.UTF_8)); RockStoreIterator iterator = new RockStoreIterator(db.newIterator()); + iterator.seekToFirst(); + Assert.assertArrayEquals("1".getBytes(StandardCharsets.UTF_8), iterator.getKey()); + Assert.assertArrayEquals("1".getBytes(StandardCharsets.UTF_8), iterator.next().getValue()); + Assert.assertTrue(iterator.hasNext()); + + Assert.assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), iterator.getValue()); + Assert.assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), iterator.next().getKey()); + Assert.assertFalse(iterator.hasNext()); + + try { + iterator.seekToLast(); + } catch (Exception e) { + Assert.assertTrue(e instanceof IllegalStateException); + } + + iterator = new RockStoreIterator(db.newIterator()); + iterator.seekToLast(); + Assert.assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), iterator.getKey()); + Assert.assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), iterator.getValue()); + iterator.seekToFirst(); while (iterator.hasNext()) { iterator.next(); } Assert.assertFalse(iterator.hasNext()); try { - iterator.next(); + iterator.getKey(); + } catch (Exception e) { + Assert.assertTrue(e instanceof IllegalStateException); + } + try { + iterator.getValue(); } catch (Exception e) { - Assert.assertTrue(e instanceof NoSuchElementException); + Assert.assertTrue(e instanceof IllegalStateException); } - db.close(); - } finally { - RocksDB.destroyDB(file.toString(), new org.rocksdb.Options()); + thrown.expect(NoSuchElementException.class); + iterator.next(); } - } } diff --git a/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java b/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java new file mode 100644 index 00000000000..17b2526d6d0 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java @@ -0,0 +1,277 @@ +package org.tron.core.services; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.tron.common.application.TronApplicationContext; +import org.tron.common.error.TronDBException; +import org.tron.common.es.ExecutorServiceManager; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.Constant; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.config.DefaultConfig; +import org.tron.core.config.args.Args; +import org.tron.core.service.MortgageService; +import org.tron.core.service.RewardViCalService; +import org.tron.core.store.AccountStore; +import org.tron.core.store.DelegationStore; +import org.tron.core.store.DynamicPropertiesStore; +import org.tron.core.store.WitnessStore; +import org.tron.protos.Protocol; + +public class ComputeRewardTest { + + private static final byte[] OWNER_ADDRESS = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1bf0"); + + private static final byte[] OWNER_ADDRESS_2 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1bf1"); + + private static final byte[] OWNER_ADDRESS_3 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1bf2"); + + private static final byte[] SR_ADDRESS_1 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c00"); + private static final byte[] SR_ADDRESS_2 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c01"); + private static final byte[] SR_ADDRESS_3 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c02"); + private static final byte[] SR_ADDRESS_4 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c03"); + private static final byte[] SR_ADDRESS_5 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c04"); + private static final byte[] SR_ADDRESS_6 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c05"); + private static final byte[] SR_ADDRESS_7 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c06"); + private static final byte[] SR_ADDRESS_8 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c07"); + private static final byte[] SR_ADDRESS_9 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c08"); + private static final byte[] SR_ADDRESS_10 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c09"); + private static final byte[] SR_ADDRESS_11 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c10"); + private static final byte[] SR_ADDRESS_12 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c11"); + private static final byte[] SR_ADDRESS_13 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c12"); + private static final byte[] SR_ADDRESS_14 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c13"); + private static final byte[] SR_ADDRESS_15 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c14"); + private static final byte[] SR_ADDRESS_16 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c15"); + private static final byte[] SR_ADDRESS_17 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c16"); + private static final byte[] SR_ADDRESS_18 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c17"); + private static final byte[] SR_ADDRESS_19 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c18"); + private static final byte[] SR_ADDRESS_20 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c19"); + private static final byte[] SR_ADDRESS_21 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c20"); + private static final byte[] SR_ADDRESS_22 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c21"); + private static final byte[] SR_ADDRESS_23 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c22"); + private static final byte[] SR_ADDRESS_24 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c23"); + private static final byte[] SR_ADDRESS_25 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c24"); + private static final byte[] SR_ADDRESS_26 = ByteArray.fromHexString( + "4105b9e8af8ee371cad87317f442d155b39fbd1c25"); + + private static TronApplicationContext context; + private static DynamicPropertiesStore propertiesStore; + private static DelegationStore delegationStore; + private static AccountStore accountStore; + private static RewardViCalService rewardViCalService; + private static WitnessStore witnessStore; + private static MortgageService mortgageService; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @After + public void destroy() { + context.destroy(); + Args.clearParam(); + } + + /** + * Init data. + */ + @Before + public void init() throws IOException { + Args.setParam(new String[]{"--output-directory", temporaryFolder.newFolder().toString(), + "--p2p-disable", "true"}, Constant.TEST_CONF); + context = new TronApplicationContext(DefaultConfig.class); + propertiesStore = context.getBean(DynamicPropertiesStore.class); + delegationStore = context.getBean(DelegationStore.class); + accountStore = context.getBean(AccountStore.class); + rewardViCalService = context.getBean(RewardViCalService.class); + witnessStore = context.getBean(WitnessStore.class); + mortgageService = context.getBean(MortgageService.class); + setUp(); + } + + private void setUp() { + // mock flush service + Map flushServices = new HashMap<>(); + flushServices.put("propertiesStore", MoreExecutors.listeningDecorator( + ExecutorServiceManager.newSingleThreadExecutor( + "flush-service-propertiesStore"))); + flushServices.put("delegationStore", MoreExecutors.listeningDecorator( + ExecutorServiceManager.newSingleThreadExecutor( + "flush-service-delegationStore"))); + flushServices.put("accountStore", MoreExecutors.listeningDecorator( + ExecutorServiceManager.newSingleThreadExecutor("flush-service-accountStore"))); + flushServices.put("witnessStore", MoreExecutors.listeningDecorator( + ExecutorServiceManager.newSingleThreadExecutor("flush-service-witnessStore"))); + + List> futures = new ArrayList<>(flushServices.size()); + + try { + flushServices.get("propertiesStore").submit(() -> { + propertiesStore.saveChangeDelegation(1); + propertiesStore.saveCurrentCycleNumber(4); + propertiesStore.saveNewRewardAlgorithmEffectiveCycle(); + propertiesStore.saveLatestBlockHeaderNumber(1); + }).get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TronDBException(e); + } catch (ExecutionException e) { + throw new TronDBException(e); + } + + try { + Thread.sleep(1000 * 6); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TronDBException(e); + } + + List votes = new ArrayList<>(32); + votes.add(new Vote(46188095536L, 5, 1496122605L, SR_ADDRESS_1)); + votes.add(new Vote(48618386224L, 5, 1582867684L, SR_ADDRESS_2)); + votes.add(new Vote(13155856728L, 5, 586969566L, SR_ADDRESS_3)); + votes.add(new Vote(41883707392L, 5, 1342484905L, SR_ADDRESS_4)); + votes.add(new Vote(62017323832L, 5, 2061119522L, SR_ADDRESS_5)); + votes.add(new Vote(19227712L, 3, 722417L, SR_ADDRESS_6)); + votes.add(new Vote(46634987592L, 3, 1599681706L, SR_ADDRESS_7)); + votes.add(new Vote(49112700L, 3, 1753127L, SR_ADDRESS_8)); + votes.add(new Vote(40835355868L, 6, 1467015537L, SR_ADDRESS_9)); + votes.add(new Vote(10045616L, 5, 362326L, SR_ADDRESS_10)); + votes.add(new Vote(34534983616L, 5, 1217718846L, SR_ADDRESS_11)); + votes.add(new Vote(32387926028L, 5, 1292557190L, SR_ADDRESS_12)); + votes.add(new Vote(36516086396L, 5, 1295716573L, SR_ADDRESS_13)); + votes.add(new Vote(48411501224L, 5, 1575483226L, SR_ADDRESS_14)); + votes.add(new Vote(154785960L, 5, 6905922L, SR_ADDRESS_15)); + votes.add(new Vote(59057915168L, 6, 1956059729L, SR_ADDRESS_16)); + votes.add(new Vote(62921824L, 3, 2245904L, SR_ADDRESS_17)); + votes.add(new Vote(1180144L, 3, 42148L, SR_ADDRESS_18)); + votes.add(new Vote(104313216L, 5, 4654248L, SR_ADDRESS_19)); + votes.add(new Vote(20429168760L, 1, 759569195L, SR_ADDRESS_20)); + votes.add(new Vote(4706184L, 3, 168069L, SR_ADDRESS_21)); + votes.add(new Vote(55804071064L, 5, 1839919389L, SR_ADDRESS_22)); + votes.add(new Vote(6074042856L, 6, 216802459L, SR_ADDRESS_23)); + votes.add(new Vote(40729360L, 5, 1817205L, SR_ADDRESS_24)); + votes.add(new Vote(31250017036L, 5, 1242358644L, SR_ADDRESS_25)); + votes.add(new Vote(15003660L, 5, 669546L, SR_ADDRESS_26)); + + futures.add(flushServices.get("delegationStore").submit(() -> { + delegationStore.setBeginCycle(OWNER_ADDRESS, 2); + delegationStore.setEndCycle(OWNER_ADDRESS, 3); + delegationStore.setBeginCycle(OWNER_ADDRESS_2, 1); + delegationStore.setEndCycle(OWNER_ADDRESS_2, 2); + delegationStore.setBeginCycle(OWNER_ADDRESS_3, 5); + for (Vote vote : votes) { + delegationStore.addReward(3, vote.srAddress, vote.totalReward); + delegationStore.setWitnessVote(3, vote.srAddress, vote.totalVotes); + } + })); + + futures.add(flushServices.get("witnessStore").submit(() -> { + for (Vote vote : votes) { + witnessStore.put(vote.srAddress, new WitnessCapsule(Protocol.Witness.newBuilder() + .setAddress(ByteString.copyFrom(vote.srAddress)) + .setVoteCount(vote.totalVotes) + .build())); + } + })); + + futures.add(flushServices.get("accountStore").submit(() -> { + Protocol.Account.Builder accountBuilder = Protocol.Account.newBuilder(); + accountBuilder.setAddress(ByteString.copyFrom(OWNER_ADDRESS)); + for (Vote vote : votes) { + accountBuilder.addVotes(Protocol.Vote.newBuilder() + .setVoteAddress(ByteString.copyFrom(vote.srAddress)) + .setVoteCount(vote.userVotes)); + + } + accountStore.put(OWNER_ADDRESS, new AccountCapsule(accountBuilder.build())); + })); + Future future = Futures.allAsList(futures); + try { + future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TronDBException(e); + } catch (ExecutionException e) { + throw new TronDBException(e); + } + try { + flushServices.get("propertiesStore").submit(() -> { + propertiesStore.saveAllowOldRewardOpt(1); + propertiesStore.saveLatestBlockHeaderNumber(3); + propertiesStore.saveCurrentCycleNumber(5); + }).get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TronDBException(e); + } catch (ExecutionException e) { + throw new TronDBException(e); + } + + rewardViCalService.setRewardViRoot(Sha256Hash.wrap( + ByteString.fromHex("e0ebe2f3243391ed674dff816a07f589a3279420d6d88bc823b6a9d5778337ce"))); + } + + @Test + public void query() { + Assert.assertEquals(3189, mortgageService.queryReward(OWNER_ADDRESS)); + } + + static class Vote { + long totalVotes; + long userVotes; + long totalReward; + byte[] srAddress; + + public Vote(long totalReward, long userVotes, long totalVotes, byte[] srAddress) { + this.totalVotes = totalVotes; + this.userVotes = userVotes; + this.totalReward = totalReward; + this.srAddress = srAddress; + } + } +} diff --git a/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java b/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java index 0ba32b27f2e..e248af647c3 100644 --- a/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java +++ b/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java @@ -1,5 +1,6 @@ package org.tron.core.services; +import static org.tron.core.utils.ProposalUtil.ProposalType.ALLOW_OLD_REWARD_OPT; import static org.tron.core.utils.ProposalUtil.ProposalType.ENERGY_FEE; import static org.tron.core.utils.ProposalUtil.ProposalType.TRANSACTION_FEE; import static org.tron.core.utils.ProposalUtil.ProposalType.WITNESS_127_PAY_PER_BLOCK; @@ -10,7 +11,9 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.tron.common.BaseTest; import org.tron.core.Constant; import org.tron.core.capsule.ProposalCapsule; @@ -22,6 +25,9 @@ @Slf4j public class ProposalServiceTest extends BaseTest { + @Rule + public ExpectedException thrown = ExpectedException.none(); + private static boolean init; @BeforeClass @@ -63,6 +69,9 @@ public void test() { proposal = Proposal.newBuilder().putParameters(proposalType.getCode(), 1).build(); } proposalCapsule = new ProposalCapsule(proposal); + if (proposalType == ALLOW_OLD_REWARD_OPT) { + thrown.expect(IllegalStateException.class); + } result = ProposalService.process(dbManager, proposalCapsule); Assert.assertTrue(result); } From 62d780d314dca7b92b784ee7323faf2953634693 Mon Sep 17 00:00:00 2001 From: liukai Date: Fri, 5 Jan 2024 18:03:37 +0800 Subject: [PATCH 26/61] test(case): add test case --- .../tron/core/db/AccountTraceStoreTest.java | 13 +++++ .../java/org/tron/core/db/BlockStoreTest.java | 50 +++++++++++++++++++ .../tron/core/db/MarketOrderStoreTest.java | 17 +++++++ .../tron/core/db/TransactionHistoryTest.java | 7 +++ 4 files changed, 87 insertions(+) diff --git a/framework/src/test/java/org/tron/core/db/AccountTraceStoreTest.java b/framework/src/test/java/org/tron/core/db/AccountTraceStoreTest.java index aa87f903ad3..5a6d44a8c7c 100644 --- a/framework/src/test/java/org/tron/core/db/AccountTraceStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/AccountTraceStoreTest.java @@ -12,6 +12,7 @@ import org.tron.common.utils.ByteArray; import org.tron.core.Constant; import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.AccountTraceCapsule; import org.tron.core.config.args.Args; import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; @@ -54,4 +55,16 @@ public void testGetPrevBalance() { Assert.assertEquals((long)pair2.getKey(),3L); Assert.assertEquals((long)pair2.getValue(), 99L); } + + @Test + public void testPut() { + long number = 2 ^ Long.MAX_VALUE; + long balance = 9999; + byte[] key = Bytes.concat(address, Longs.toByteArray(number)); + accountTraceStore.put(key, new AccountTraceCapsule(balance)); + Pair pair = accountTraceStore.getPrevBalance(address,2); + Assert.assertEquals((long)pair.getKey(),2L); + Assert.assertEquals((long)pair.getValue(), 0L); + } + } \ No newline at end of file diff --git a/framework/src/test/java/org/tron/core/db/BlockStoreTest.java b/framework/src/test/java/org/tron/core/db/BlockStoreTest.java index 4f706a97dcf..23ff8ab75b7 100644 --- a/framework/src/test/java/org/tron/core/db/BlockStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/BlockStoreTest.java @@ -1,21 +1,71 @@ package org.tron.core.db; +import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; import org.junit.Test; import org.tron.common.BaseTest; +import org.tron.common.utils.JsonUtilTest; +import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; +import org.tron.core.capsule.BlockCapsule; import org.tron.core.config.args.Args; +import org.tron.core.exception.BadItemException; +import org.tron.core.exception.ItemNotFoundException; + +import javax.annotation.Resource; @Slf4j public class BlockStoreTest extends BaseTest { + @Resource + private BlockStore blockStore; static { Args.setParam(new String[]{"--output-directory", dbPath()}, Constant.TEST_CONF); } + private BlockCapsule getBlockCapsule(long number) { + return new BlockCapsule(number, Sha256Hash.ZERO_HASH, + System.currentTimeMillis(), ByteString.EMPTY); + } + @Test public void testCreateBlockStore() { } + + @Test + public void testPut() { + long number = 1; + BlockCapsule blockCapsule = getBlockCapsule(number); + + byte[] blockId = blockCapsule.getBlockId().getBytes(); + blockStore.put(blockId, blockCapsule); + try { + BlockCapsule blockCapsule1 = blockStore.get(blockId); + Assert.assertNotNull(blockCapsule1); + Assert.assertEquals(number, blockCapsule1.getNum()); + } catch (ItemNotFoundException | BadItemException e) { + e.printStackTrace(); + } + } + + @Test + public void testGet() { + long number = 2; + BlockCapsule blockCapsule = getBlockCapsule(number); + byte[] blockId = blockCapsule.getBlockId().getBytes(); + blockStore.put(blockId, blockCapsule); + try { + boolean has = blockStore.has(blockId); + Assert.assertTrue(has); + BlockCapsule blockCapsule1 = blockStore.get(blockId); + + Assert.assertEquals(number, blockCapsule1.getNum()); + } catch (ItemNotFoundException | BadItemException e) { + e.printStackTrace(); + } + } + } diff --git a/framework/src/test/java/org/tron/core/db/MarketOrderStoreTest.java b/framework/src/test/java/org/tron/core/db/MarketOrderStoreTest.java index 33126783e9a..1cfdb20da97 100644 --- a/framework/src/test/java/org/tron/core/db/MarketOrderStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/MarketOrderStoreTest.java @@ -8,6 +8,7 @@ import org.tron.core.Constant; import org.tron.core.capsule.MarketOrderCapsule; import org.tron.core.config.args.Args; +import org.tron.core.exception.ItemNotFoundException; import org.tron.core.store.MarketOrderStore; import org.tron.protos.Protocol; @@ -44,4 +45,20 @@ public void testGet() throws Exception { Assert.assertEquals(result.getBuyTokenQuantity(), 100L); } + @Test + public void testDelete() throws ItemNotFoundException { + byte[] orderId = "testDelete".getBytes(); + marketOrderStore.put(orderId, + new MarketOrderCapsule(Protocol.MarketOrder.newBuilder() + .setOrderId(ByteString.copyFrom(orderId)) + .setSellTokenId(ByteString.copyFrom("addr1".getBytes())) + .setSellTokenQuantity(200L) + .setBuyTokenId(ByteString.copyFrom("addr2".getBytes())) + .setBuyTokenQuantity(100L) + .build())); + marketOrderStore.delete(orderId); + final MarketOrderCapsule result = marketOrderStore.getUnchecked(orderId); + Assert.assertNull(result); + } + } diff --git a/framework/src/test/java/org/tron/core/db/TransactionHistoryTest.java b/framework/src/test/java/org/tron/core/db/TransactionHistoryTest.java index eef168938b2..c5c249b6f70 100644 --- a/framework/src/test/java/org/tron/core/db/TransactionHistoryTest.java +++ b/framework/src/test/java/org/tron/core/db/TransactionHistoryTest.java @@ -59,4 +59,11 @@ public void get() throws BadItemException { Assert.assertEquals(ByteArray.toHexString(transactionId), ByteArray.toHexString(resultCapsule.getId())); } + + @Test + public void testDelete() throws BadItemException { + transactionHistoryStore.delete(transactionId); + TransactionInfoCapsule transactionInfoCapsule = transactionHistoryStore.get(transactionId); + Assert.assertNull(transactionInfoCapsule); + } } \ No newline at end of file From ae1865ec3ed3678ba2c04e19c9aa8e0b17490786 Mon Sep 17 00:00:00 2001 From: liukai Date: Fri, 5 Jan 2024 19:52:47 +0800 Subject: [PATCH 27/61] test(case): fix import order --- framework/src/test/java/org/tron/core/db/BlockStoreTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/framework/src/test/java/org/tron/core/db/BlockStoreTest.java b/framework/src/test/java/org/tron/core/db/BlockStoreTest.java index 23ff8ab75b7..d1ec0ae4f88 100644 --- a/framework/src/test/java/org/tron/core/db/BlockStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/BlockStoreTest.java @@ -1,11 +1,11 @@ package org.tron.core.db; import com.google.protobuf.ByteString; +import javax.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.Test; import org.tron.common.BaseTest; -import org.tron.common.utils.JsonUtilTest; import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; import org.tron.core.capsule.BlockCapsule; @@ -13,7 +13,6 @@ import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; -import javax.annotation.Resource; @Slf4j public class BlockStoreTest extends BaseTest { From f04616ac048af1b90ce3f22c5d1eae7334b09039 Mon Sep 17 00:00:00 2001 From: liukai Date: Sat, 6 Jan 2024 23:46:37 +0800 Subject: [PATCH 28/61] test(case): add case --- .../java/org/tron/core/db/BlockStoreTest.java | 20 ++++++++++++++ .../core/db/DelegatedResourceStoreTest.java | 15 +++++++++++ .../org/tron/core/db/DelegationStoreTest.java | 17 ++++++++++++ .../org/tron/core/db/ExchangeStoreTest.java | 25 ++++++++++++++++++ .../org/tron/core/db/ExchangeV2StoreTest.java | 26 +++++++++++++++++++ .../tron/core/db/MarketAccountStoreTest.java | 15 +++++++++++ 6 files changed, 118 insertions(+) diff --git a/framework/src/test/java/org/tron/core/db/BlockStoreTest.java b/framework/src/test/java/org/tron/core/db/BlockStoreTest.java index d1ec0ae4f88..937a102193f 100644 --- a/framework/src/test/java/org/tron/core/db/BlockStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/BlockStoreTest.java @@ -67,4 +67,24 @@ public void testGet() { } } + @Test + public void testDelete() { + long number = 1; + BlockCapsule blockCapsule = getBlockCapsule(number); + + byte[] blockId = blockCapsule.getBlockId().getBytes(); + blockStore.put(blockId, blockCapsule); + try { + BlockCapsule blockCapsule1 = blockStore.get(blockId); + Assert.assertNotNull(blockCapsule1); + Assert.assertEquals(number, blockCapsule1.getNum()); + + blockStore.delete(blockId); + BlockCapsule blockCapsule2 = blockStore.getUnchecked(blockId); + Assert.assertNull(blockCapsule2); + } catch (ItemNotFoundException | BadItemException e) { + e.printStackTrace(); + } + } + } diff --git a/framework/src/test/java/org/tron/core/db/DelegatedResourceStoreTest.java b/framework/src/test/java/org/tron/core/db/DelegatedResourceStoreTest.java index 8878ff0dcf3..905ef0384f1 100644 --- a/framework/src/test/java/org/tron/core/db/DelegatedResourceStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/DelegatedResourceStoreTest.java @@ -57,6 +57,21 @@ public void testPut() { Assert.assertEquals(BALANCE, delegatedResourceCapsule1.getFrozenBalanceForEnergy()); } + @Test + public void testDelete() { + DelegatedResourceCapsule delegatedResourceCapsule = create("444444444"); + byte[] key = delegatedResourceCapsule.createDbKey(); + delegatedResourceStore.put(key, delegatedResourceCapsule); + DelegatedResourceCapsule delegatedResourceCapsule1 = delegatedResourceStore.get(key); + Assert.assertNotNull(delegatedResourceCapsule1); + Assert.assertEquals(BALANCE, delegatedResourceCapsule1.getFrozenBalanceForEnergy()); + + delegatedResourceStore.delete(key); + DelegatedResourceCapsule delegatedResourceCapsule2 = delegatedResourceStore.get(key); + Assert.assertNull(delegatedResourceCapsule2); + + } + public DelegatedResourceCapsule create(String address) { byte[] ownerAddress = ByteArray.fromHexString(address); byte[] receiverAddress = ByteArray.fromHexString(RECEIVER_ADDRESS); diff --git a/framework/src/test/java/org/tron/core/db/DelegationStoreTest.java b/framework/src/test/java/org/tron/core/db/DelegationStoreTest.java index d63015ae064..10e70a0a83b 100644 --- a/framework/src/test/java/org/tron/core/db/DelegationStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/DelegationStoreTest.java @@ -68,4 +68,21 @@ public void testPut() { Assert.assertEquals(value, actualValue); } + @Test + public void testDelete() { + long value = 20_000_000; + byte[] key = buildRewardKey(CYCLE, ByteArray.fromHexString("33333333")); + delegationStore.put(key, new BytesCapsule(ByteArray + .fromLong(20_000_000))); + + BytesCapsule bytesCapsule = delegationStore.get(key); + Assert.assertNotNull(bytesCapsule); + long actualValue = ByteArray.toLong(bytesCapsule.getData()); + Assert.assertEquals(value, actualValue); + + delegationStore.delete(key); + BytesCapsule bytesCapsule1 = delegationStore.getUnchecked(key); + Assert.assertNull(bytesCapsule1.getData()); + } + } diff --git a/framework/src/test/java/org/tron/core/db/ExchangeStoreTest.java b/framework/src/test/java/org/tron/core/db/ExchangeStoreTest.java index f48ac11301d..685c137422c 100644 --- a/framework/src/test/java/org/tron/core/db/ExchangeStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/ExchangeStoreTest.java @@ -10,6 +10,7 @@ import org.tron.core.Constant; import org.tron.core.capsule.ExchangeCapsule; import org.tron.core.config.args.Args; +import org.tron.core.exception.ItemNotFoundException; import org.tron.core.store.ExchangeStore; import org.tron.protos.Protocol; @@ -50,6 +51,30 @@ public void testGet() throws Exception { Assert.assertEquals(result.getID(), 1); } + @Test + public void testPut() throws ItemNotFoundException { + Protocol.Exchange.Builder builder = Protocol.Exchange.newBuilder(); + builder.setExchangeId(1L).setCreatorAddress(ByteString.copyFromUtf8("Address1")); + ExchangeCapsule exchangeCapsule = new ExchangeCapsule(builder.build()); + exchangeKey1 = exchangeCapsule.createDbKey(); + chainBaseManager.getExchangeStore().put(exchangeKey1, exchangeCapsule); + + final ExchangeCapsule result = exchangeStore.get(exchangeKey1); + Assert.assertNotNull(result); + Assert.assertEquals(result.getID(), 1); + } + + @Test + public void testDelete() throws Exception { + final ExchangeCapsule result = exchangeStore.get(exchangeKey1); + Assert.assertNotNull(result); + Assert.assertEquals(result.getID(), 1); + + exchangeStore.delete(exchangeKey1); + ExchangeCapsule exchangeCapsule = exchangeStore.getUnchecked(exchangeKey1); + Assert.assertNull(exchangeCapsule); + } + @Test public void testGetAllExchanges() { List exchangeCapsuleList = exchangeStore.getAllExchanges(); diff --git a/framework/src/test/java/org/tron/core/db/ExchangeV2StoreTest.java b/framework/src/test/java/org/tron/core/db/ExchangeV2StoreTest.java index 2b1e2ea31ea..97c5f599b6e 100644 --- a/framework/src/test/java/org/tron/core/db/ExchangeV2StoreTest.java +++ b/framework/src/test/java/org/tron/core/db/ExchangeV2StoreTest.java @@ -8,6 +8,7 @@ import org.tron.core.Constant; import org.tron.core.capsule.ExchangeCapsule; import org.tron.core.config.args.Args; +import org.tron.core.exception.ItemNotFoundException; import org.tron.core.store.ExchangeV2Store; import org.tron.protos.Protocol; @@ -33,6 +34,31 @@ public void testGet() throws Exception { Assert.assertEquals(result.getID(), 1); } + @Test + public void testPut() throws ItemNotFoundException { + Protocol.Exchange.Builder builder = Protocol.Exchange.newBuilder().setExchangeId(1L) + .setCreatorAddress(ByteString.copyFromUtf8("Address2")); + ExchangeCapsule exchangeCapsule = new ExchangeCapsule(builder.build()); + byte[] exchangeKey1 = exchangeCapsule.createDbKey(); + exchangeV2Store.put(exchangeKey1, exchangeCapsule); + + final ExchangeCapsule result = exchangeV2Store.get(exchangeKey1); + Assert.assertNotNull(result); + Assert.assertEquals(result.getID(), 1); + } + + @Test + public void testDelete() throws ItemNotFoundException { + Protocol.Exchange.Builder builder = Protocol.Exchange.newBuilder().setExchangeId(1L) + .setCreatorAddress(ByteString.copyFromUtf8("Address3")); + ExchangeCapsule exchangeCapsule = new ExchangeCapsule(builder.build()); + byte[] exchangeKey1 = exchangeCapsule.createDbKey(); + exchangeV2Store.put(exchangeKey1, exchangeCapsule); + exchangeV2Store.delete(exchangeKey1); + ExchangeCapsule result = exchangeV2Store.getUnchecked(exchangeKey1); + Assert.assertNull(result); + } + private byte[] putToExchangeV2() { Protocol.Exchange.Builder builder = Protocol.Exchange.newBuilder().setExchangeId(1L) .setCreatorAddress(ByteString.copyFromUtf8("Address1")); diff --git a/framework/src/test/java/org/tron/core/db/MarketAccountStoreTest.java b/framework/src/test/java/org/tron/core/db/MarketAccountStoreTest.java index 51bf1c5d7fc..87bd70b875a 100644 --- a/framework/src/test/java/org/tron/core/db/MarketAccountStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/MarketAccountStoreTest.java @@ -8,6 +8,7 @@ import org.tron.core.Constant; import org.tron.core.capsule.MarketAccountOrderCapsule; import org.tron.core.config.args.Args; +import org.tron.core.exception.ItemNotFoundException; import org.tron.core.store.MarketAccountStore; public class MarketAccountStoreTest extends BaseTest { @@ -27,6 +28,10 @@ public class MarketAccountStoreTest extends BaseTest { @Test public void testGet() throws Exception { String address = "Address1"; + + MarketAccountOrderCapsule marketAccountOrderCapsule = marketAccountStore.getUnchecked(address.getBytes()); + Assert.assertNull(marketAccountOrderCapsule); + marketAccountStore.put(address.getBytes(), new MarketAccountOrderCapsule(ByteString.copyFrom(address.getBytes()))); final MarketAccountOrderCapsule result = marketAccountStore.get(address.getBytes()); @@ -34,4 +39,14 @@ public void testGet() throws Exception { Assert.assertEquals(result.getOwnerAddress(), ByteString.copyFrom(address.getBytes())); } + @Test + public void testPut() throws ItemNotFoundException { + String address = "Address1"; + marketAccountStore.put(address.getBytes(), + new MarketAccountOrderCapsule(ByteString.copyFrom(address.getBytes()))); + final MarketAccountOrderCapsule result = marketAccountStore.get(address.getBytes()); + Assert.assertNotNull(result); + Assert.assertEquals(result.getOwnerAddress(), ByteString.copyFrom(address.getBytes())); + } + } From 8107586127ba19e6b05f51be1b93476cc0d8c522 Mon Sep 17 00:00:00 2001 From: liukai Date: Sun, 7 Jan 2024 15:04:14 +0800 Subject: [PATCH 29/61] test(case): add ZKProofStoreTest case --- .../tron/core/db/MarketAccountStoreTest.java | 3 +- .../org/tron/core/db/ZKProofStoreTest.java | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 framework/src/test/java/org/tron/core/db/ZKProofStoreTest.java diff --git a/framework/src/test/java/org/tron/core/db/MarketAccountStoreTest.java b/framework/src/test/java/org/tron/core/db/MarketAccountStoreTest.java index 87bd70b875a..ed94a64175d 100644 --- a/framework/src/test/java/org/tron/core/db/MarketAccountStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/MarketAccountStoreTest.java @@ -29,7 +29,8 @@ public class MarketAccountStoreTest extends BaseTest { public void testGet() throws Exception { String address = "Address1"; - MarketAccountOrderCapsule marketAccountOrderCapsule = marketAccountStore.getUnchecked(address.getBytes()); + MarketAccountOrderCapsule marketAccountOrderCapsule = + marketAccountStore.getUnchecked(address.getBytes()); Assert.assertNull(marketAccountOrderCapsule); marketAccountStore.put(address.getBytes(), diff --git a/framework/src/test/java/org/tron/core/db/ZKProofStoreTest.java b/framework/src/test/java/org/tron/core/db/ZKProofStoreTest.java new file mode 100644 index 00000000000..a8aa07c4342 --- /dev/null +++ b/framework/src/test/java/org/tron/core/db/ZKProofStoreTest.java @@ -0,0 +1,58 @@ +package org.tron.core.db; + +import com.google.protobuf.ByteString; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.capsule.TransactionCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.store.ZKProofStore; +import org.tron.protos.Protocol; +import org.tron.protos.contract.BalanceContract; + +public class ZKProofStoreTest extends BaseTest { + + static { + Args.setParam(new String[]{"--output-directory", dbPath()}, + Constant.TEST_CONF); + } + + @Autowired + private ZKProofStore proofStore; + + private TransactionCapsule getTransactionCapsule() { + BalanceContract.TransferContract transferContract = + BalanceContract.TransferContract.newBuilder() + .setAmount(10) + .setOwnerAddress(ByteString.copyFromUtf8("aaa")) + .setToAddress(ByteString.copyFromUtf8("bbb")) + .build(); + return new TransactionCapsule(transferContract, + Protocol.Transaction.Contract.ContractType.TransferContract); + } + + @Test + public void testPut() { + TransactionCapsule trx = getTransactionCapsule(); + proofStore.put(trx.getTransactionId().getBytes(), + true); + boolean has = proofStore.has(trx.getTransactionId().getBytes()); + Assert.assertTrue(has); + } + + @Test + public void testGet() { + TransactionCapsule trx = getTransactionCapsule(); + proofStore.put(trx.getTransactionId().getBytes(), + true); + Boolean result = proofStore.get(trx.getTransactionId().getBytes()); + Assert.assertEquals(true, result); + + proofStore.put(trx.getTransactionId().getBytes(), + false); + result = proofStore.get(trx.getTransactionId().getBytes()); + Assert.assertEquals(false, result); + } +} From 8721f800a224d34b55e882d7eeb4c744ff91aa60 Mon Sep 17 00:00:00 2001 From: liukai Date: Mon, 8 Jan 2024 17:45:36 +0800 Subject: [PATCH 30/61] test(case): add servlet case --- .../http/GetAccountByIdServletTest.java | 47 +++++++++++++++ .../http/GetBlockByIdServletTest.java | 49 ++++++++++++++++ .../http/GetBlockByNumServletTest.java | 58 +++++++++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 framework/src/test/java/org/tron/core/services/http/GetAccountByIdServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/GetBlockByIdServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/GetBlockByNumServletTest.java diff --git a/framework/src/test/java/org/tron/core/services/http/GetAccountByIdServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetAccountByIdServletTest.java new file mode 100644 index 00000000000..38c23a971ff --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetAccountByIdServletTest.java @@ -0,0 +1,47 @@ +package org.tron.core.services.http; + +import static org.apache.commons.lang3.StringUtils.isNotEmpty; + +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class GetAccountByIdServletTest extends BaseTest { + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Resource + private GetAccountByIdServlet getAccountByIdServlet; + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + if (isNotEmpty(contentType)) { + request.setContentType(contentType); + } + request.setCharacterEncoding("UTF-8"); + return request; + } + + @Test + public void testGetAccountById() { + String jsonParam = "{\"account_id\": \"6161616162626262\"}"; + MockHttpServletRequest request = createRequest("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + + getAccountByIdServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/GetBlockByIdServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetBlockByIdServletTest.java new file mode 100644 index 00000000000..f283035ca1d --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetBlockByIdServletTest.java @@ -0,0 +1,49 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; + +import javax.annotation.Resource; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class GetBlockByIdServletTest extends BaseTest { + + @Resource + private GetBlockByIdServlet getBlockByIdServlet; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + if (isNotEmpty(contentType)) { + request.setContentType(contentType); + } + request.setCharacterEncoding("UTF-8"); + return request; + } + + @Test + public void testGetBlockById() { + String jsonParam = "{\"value\": " + + "\"0000000002951a2f65db6725c2d0583f1ab9bdb1520eeedece99d9c98f3\"}"; + MockHttpServletRequest request = createRequest("application/json"); + request.setContent(jsonParam.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + getBlockByIdServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/GetBlockByNumServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetBlockByNumServletTest.java new file mode 100644 index 00000000000..d3ffa735d8d --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetBlockByNumServletTest.java @@ -0,0 +1,58 @@ +package org.tron.core.services.http; + +import static org.apache.commons.lang3.StringUtils.isNotEmpty; +import static org.junit.Assert.assertTrue; + +import com.alibaba.fastjson.JSONObject; + +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class GetBlockByNumServletTest extends BaseTest { + + @Resource + private GetBlockByNumServlet getBlockByNumServlet; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + public MockHttpServletRequest createRequest(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod("POST"); + if (isNotEmpty(contentType)) { + request.setContentType(contentType); + } + request.setCharacterEncoding("UTF-8"); + return request; + } + + @Test + public void testGetBlockByNum() { + String jsonParam = "{\"number\": 1}"; + MockHttpServletRequest request = createRequest("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + + try { + getBlockByNumServlet.doPost(request, response); + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + assertTrue(result.containsKey("blockID")); + assertTrue(result.containsKey("transactions")); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + +} From 05e3f5553bbed6404351be3365c41e18e941b5f0 Mon Sep 17 00:00:00 2001 From: liukai Date: Tue, 9 Jan 2024 16:59:42 +0800 Subject: [PATCH 31/61] test(case): add some cases 1. add ClearABIServletTest 2. add CreateAccountServletTest 3. CreateAssetIssueServletTest --- .../services/http/ClearABIServletTest.java | 45 ++++++++++++++ .../http/CreateAccountServletTest.java | 46 ++++++++++++++ .../http/CreateAssetIssueServletTest.java | 61 +++++++++++++++++++ .../http/GetBlockByIdServletTest.java | 26 ++++---- .../http/GetBlockByNumServletTest.java | 29 ++++++--- 5 files changed, 186 insertions(+), 21 deletions(-) create mode 100644 framework/src/test/java/org/tron/core/services/http/ClearABIServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/CreateAccountServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/CreateAssetIssueServletTest.java diff --git a/framework/src/test/java/org/tron/core/services/http/ClearABIServletTest.java b/framework/src/test/java/org/tron/core/services/http/ClearABIServletTest.java new file mode 100644 index 00000000000..27bc2a6eb34 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/ClearABIServletTest.java @@ -0,0 +1,45 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import javax.annotation.Resource; + +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class ClearABIServletTest extends BaseTest { + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Resource + private ClearABIServlet clearABIServlet; + + @Test + public void testClear() { + String jsonParam = "{\n" + + " \"owner_address\": \"41a7d8a35b260395c14aa456297662092ba3b76fc0\",\n" + + " \"contract_address\": \"417bcb781f4743afaacf9f9528f3ea903b3782339f\"\n" + + "}"; + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes(UTF_8)); + + MockHttpServletResponse response = new MockHttpServletResponse(); + clearABIServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + } + +} diff --git a/framework/src/test/java/org/tron/core/services/http/CreateAccountServletTest.java b/framework/src/test/java/org/tron/core/services/http/CreateAccountServletTest.java new file mode 100644 index 00000000000..70a319e6040 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/CreateAccountServletTest.java @@ -0,0 +1,46 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import javax.annotation.Resource; + +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + + +public class CreateAccountServletTest extends BaseTest { + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Resource + private CreateAccountServlet createAccountServlet; + + @Test + public void testCreate() { + String jsonParam = "{" + + "\"owner_address\": \"41d1e7a6bc354106cb410e65ff8b181c600ff14292\"," + + "\"account_address\": \"41e552f6487585c2b58bc2c9bb4492bc1f17132cd0\"" + + "}"; + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes(UTF_8)); + + MockHttpServletResponse response = new MockHttpServletResponse(); + createAccountServlet.doPost(request, response); + + Assert.assertEquals(200, response.getStatus()); + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/CreateAssetIssueServletTest.java b/framework/src/test/java/org/tron/core/services/http/CreateAssetIssueServletTest.java new file mode 100644 index 00000000000..d9f9ccc412c --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/CreateAssetIssueServletTest.java @@ -0,0 +1,61 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import javax.annotation.Resource; + +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class CreateAssetIssueServletTest extends BaseTest { + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + + @Resource + private CreateAssetIssueServlet createAssetIssueServlet; + + @Test + public void testCreate() { + String jsonParam = "{" + + " \"owner_address\": \"41e552f6487585c2b58bc2c9bb4492bc1f17132cd0\"," + + " \"name\": \"0x6173736574497373756531353330383934333132313538\"," + + " \"abbr\": \"0x6162627231353330383934333132313538\"," + + " \"total_supply\": 4321," + + " \"trx_num\": 1," + + " \"num\": 1," + + " \"start_time\": 1530894315158," + + " \"end_time\": 1533894312158," + + " \"description\": \"007570646174654e616d6531353330363038383733343633\"," + + " \"url\": \"007570646174654e616d6531353330363038383733343633\"," + + " \"free_asset_net_limit\": 10000," + + " \"public_free_asset_net_limit\": 10000," + + " \"frozen_supply\": {" + + " \"frozen_amount\": 1," + + " \"frozen_days\": 2" + + " }" + + "}"; + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes(UTF_8)); + + MockHttpServletResponse response = new MockHttpServletResponse(); + createAssetIssueServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + + } + +} diff --git a/framework/src/test/java/org/tron/core/services/http/GetBlockByIdServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetBlockByIdServletTest.java index f283035ca1d..8b213e12640 100644 --- a/framework/src/test/java/org/tron/core/services/http/GetBlockByIdServletTest.java +++ b/framework/src/test/java/org/tron/core/services/http/GetBlockByIdServletTest.java @@ -1,10 +1,12 @@ package org.tron.core.services.http; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.commons.lang3.StringUtils.isNotEmpty; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; import javax.annotation.Resource; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; import org.junit.Assert; import org.junit.Test; import org.springframework.mock.web.MockHttpServletRequest; @@ -26,24 +28,24 @@ public class GetBlockByIdServletTest extends BaseTest { ); } - public MockHttpServletRequest createRequest(String contentType) { - MockHttpServletRequest request = new MockHttpServletRequest(); - request.setMethod("POST"); - if (isNotEmpty(contentType)) { - request.setContentType(contentType); - } - request.setCharacterEncoding("UTF-8"); - return request; - } - @Test public void testGetBlockById() { String jsonParam = "{\"value\": " + "\"0000000002951a2f65db6725c2d0583f1ab9bdb1520eeedece99d9c98f3\"}"; - MockHttpServletRequest request = createRequest("application/json"); + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); request.setContent(jsonParam.getBytes(UTF_8)); MockHttpServletResponse response = new MockHttpServletResponse(); getBlockByIdServlet.doPost(request, response); Assert.assertEquals(200, response.getStatus()); } + + @Test + public void testGet() { + MockHttpServletRequest request = createRequest(HttpGet.METHOD_NAME); + request.addParameter("value", "0000000002951a2f65db6725c2d0583f1ab9bdb1520eeedece99d9c98f3"); + MockHttpServletResponse response = new MockHttpServletResponse(); + getBlockByIdServlet.doGet(request, response); + Assert.assertEquals(200, response.getStatus()); + } } diff --git a/framework/src/test/java/org/tron/core/services/http/GetBlockByNumServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetBlockByNumServletTest.java index d3ffa735d8d..5ff84c54dbe 100644 --- a/framework/src/test/java/org/tron/core/services/http/GetBlockByNumServletTest.java +++ b/framework/src/test/java/org/tron/core/services/http/GetBlockByNumServletTest.java @@ -1,12 +1,14 @@ package org.tron.core.services.http; -import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.junit.Assert.assertTrue; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; import com.alibaba.fastjson.JSONObject; import java.io.UnsupportedEncodingException; import javax.annotation.Resource; + +import org.apache.http.client.methods.HttpPost; import org.junit.Test; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; @@ -27,18 +29,27 @@ public class GetBlockByNumServletTest extends BaseTest { ); } - public MockHttpServletRequest createRequest(String contentType) { - MockHttpServletRequest request = new MockHttpServletRequest(); - request.setMethod("POST"); - if (isNotEmpty(contentType)) { - request.setContentType(contentType); + @Test + public void testGetBlockByNum() { + String jsonParam = "{\"number\": 1}"; + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + + try { + getBlockByNumServlet.doPost(request, response); + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + assertTrue(result.containsKey("blockID")); + assertTrue(result.containsKey("transactions")); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); } - request.setCharacterEncoding("UTF-8"); - return request; } @Test - public void testGetBlockByNum() { + public void testGet() { String jsonParam = "{\"number\": 1}"; MockHttpServletRequest request = createRequest("application/json"); request.setContent(jsonParam.getBytes()); From bc701dec121baa5c8234a487a7ce7c92fc705ef1 Mon Sep 17 00:00:00 2001 From: liukai Date: Wed, 10 Jan 2024 15:30:52 +0800 Subject: [PATCH 32/61] test(case): add db test cases --- .../tron/core/db/RecentBlockStoreTest.java | 81 +++++++++++++++++ .../core/db/RecentTransactionStoreTest.java | 72 +++++++++++++++ .../AccountPermissionUpdateServletTest.java | 88 +++++++++++++++++++ .../http/UpdateAccountServletTest.java | 77 ++++++++++++++++ 4 files changed, 318 insertions(+) create mode 100644 framework/src/test/java/org/tron/core/db/RecentBlockStoreTest.java create mode 100644 framework/src/test/java/org/tron/core/db/RecentTransactionStoreTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/AccountPermissionUpdateServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/UpdateAccountServletTest.java diff --git a/framework/src/test/java/org/tron/core/db/RecentBlockStoreTest.java b/framework/src/test/java/org/tron/core/db/RecentBlockStoreTest.java new file mode 100644 index 00000000000..7856fe337a5 --- /dev/null +++ b/framework/src/test/java/org/tron/core/db/RecentBlockStoreTest.java @@ -0,0 +1,81 @@ +package org.tron.core.db; + +import com.google.protobuf.ByteString; +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.Constant; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.capsule.BytesCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.exception.ItemNotFoundException; + +public class RecentBlockStoreTest extends BaseTest { + + @Resource + private RecentBlockStore recentBlockStore; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath() + }, + Constant.TEST_CONF + ); + } + + private BlockCapsule getBlockCapsule() { + long number = 1; + return new BlockCapsule(number, + Sha256Hash.ZERO_HASH, + System.currentTimeMillis(), + ByteString.EMPTY); + } + + @Test + public void testPut() { + BlockCapsule blockCapsule = getBlockCapsule(); + byte[] key = ByteArray.subArray( + ByteArray.fromLong(blockCapsule.getNum()), 6, 8); + recentBlockStore.put(key, + new BytesCapsule(ByteArray.subArray(blockCapsule + .getBlockId().getBytes(), + 8, + 16))); + + Assert.assertTrue(recentBlockStore.has(key)); + } + + @Test + public void testGet() throws ItemNotFoundException { + BlockCapsule blockCapsule = getBlockCapsule(); + byte[] key = ByteArray.subArray( + ByteArray.fromLong(blockCapsule.getNum()), 6, 8); + BytesCapsule value = new BytesCapsule(ByteArray + .subArray(blockCapsule.getBlockId().getBytes(), + 8, + 16)); + recentBlockStore.put(key, value); + + BytesCapsule bytesCapsule = recentBlockStore.get(key); + Assert.assertNotNull(bytesCapsule); + Assert.assertArrayEquals(value.getData(), bytesCapsule.getData()); + } + + @Test + public void testDelete() { + BlockCapsule blockCapsule = getBlockCapsule(); + byte[] key = ByteArray.subArray( + ByteArray.fromLong(blockCapsule.getNum()), 6, 8); + recentBlockStore.put(key, + new BytesCapsule(ByteArray.subArray(blockCapsule + .getBlockId().getBytes(), + 8, + 16))); + recentBlockStore.delete(key); + Assert.assertFalse(recentBlockStore.has(key)); + } +} diff --git a/framework/src/test/java/org/tron/core/db/RecentTransactionStoreTest.java b/framework/src/test/java/org/tron/core/db/RecentTransactionStoreTest.java new file mode 100644 index 00000000000..20447dfc6a1 --- /dev/null +++ b/framework/src/test/java/org/tron/core/db/RecentTransactionStoreTest.java @@ -0,0 +1,72 @@ +package org.tron.core.db; + +import com.google.protobuf.ByteString; +import javax.annotation.Resource; + +import org.junit.Assert; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; +import org.tron.core.Constant; +import org.tron.core.capsule.BytesCapsule; +import org.tron.core.capsule.TransactionCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.exception.ItemNotFoundException; +import org.tron.protos.Protocol; +import org.tron.protos.contract.BalanceContract; + +public class RecentTransactionStoreTest extends BaseTest { + + @Resource + private RecentTransactionStore recentTransactionStore; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath() + }, + Constant.TEST_CONF + ); + } + + private TransactionCapsule createTransaction() { + BalanceContract.TransferContract tc = + BalanceContract.TransferContract.newBuilder() + .setAmount(10) + .setOwnerAddress(ByteString.copyFromUtf8("aaa")) + .setToAddress(ByteString.copyFromUtf8("bbb")) + .build(); + return new TransactionCapsule(tc, + Protocol.Transaction.Contract.ContractType.TransferContract); + } + + + @Test + public void testPut() { + TransactionCapsule transaction = createTransaction(); + byte[] key = transaction.getTransactionId().getBytes(); + BytesCapsule value = new BytesCapsule(ByteArray.subArray(transaction + .getTransactionId().getBytes(), + 8, + 16)); + recentTransactionStore.put(key, value); + Assert.assertTrue(recentTransactionStore.has(key)); + } + + @Test + public void testGet() throws ItemNotFoundException { + TransactionCapsule transaction = createTransaction(); + byte[] key = transaction.getTransactionId().getBytes(); + BytesCapsule value = new BytesCapsule( + ByteArray.subArray(transaction + .getTransactionId().getBytes(), + 8, + 16)); + recentTransactionStore.put(key, value); + + BytesCapsule bytesCapsule = recentTransactionStore.get(key); + Assert.assertNotNull(bytesCapsule); + Assert.assertArrayEquals(value.getData(), bytesCapsule.getData()); + + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/AccountPermissionUpdateServletTest.java b/framework/src/test/java/org/tron/core/services/http/AccountPermissionUpdateServletTest.java new file mode 100644 index 00000000000..313a0a12a60 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/AccountPermissionUpdateServletTest.java @@ -0,0 +1,88 @@ +package org.tron.core.services.http; + +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import javax.annotation.Resource; + +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class AccountPermissionUpdateServletTest extends BaseTest { + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Resource + private AccountPermissionUpdateServlet accountPermissionUpdateServlet; + + private String getParam() { + return "{" + + " \"owner_address\":\"TRGhNNfnmgLegT4zHNjEqDSADjgmnHvubJ\"," + + " \"owner\":{" + + " \"type\":0," + + " \"permission_name\":\"owner\"," + + " \"threshold\":1," + + " \"keys\":[" + + " {" + + " \"address\":\"TRGhNNfnmgLegT4zHNjEqDSADjgmnHvubJ\"," + + " \"weight\":1" + + " }" + + " ]" + + " }," + + " \"witness\":{" + + " \"type\":1," + + " \"permission_name\":\"witness\"," + + " \"threshold\":1," + + " \"keys\":[" + + " {" + + " \"address\":\"TRGhNNfnmgLegT4zHNjEqDSADjgmnHvubJ\"," + + " \"weight\":1" + + " }" + + " ]" + + " }," + + " \"actives\":[" + + " {" + + " \"type\":2," + + " \"permission_name\":\"active12323\"," + + " \"threshold\":2," + + " \"operations\":" + + "\"7fff1fc0033e0000000000000000000000000000000000000000000000000000\"," + + " \"keys\":[" + + " {" + + " \"address\":\"TNhXo1GbRNCuorvYu5JFWN3m2NYr9QQpVR\"," + + " \"weight\":1" + + " }," + + " {" + + " \"address\":\"TKwhcDup8L2PH5r6hxp5CQvQzZqJLmKvZP\"," + + " \"weight\":1" + + " }" + + " ]" + + " }" + + " ]," + + " \"visible\": true" + + "}"; + } + + @Test + public void test() { + String jsonParam = getParam(); + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + + accountPermissionUpdateServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/UpdateAccountServletTest.java b/framework/src/test/java/org/tron/core/services/http/UpdateAccountServletTest.java new file mode 100644 index 00000000000..e5064e2013b --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/UpdateAccountServletTest.java @@ -0,0 +1,77 @@ +package org.tron.core.services.http; + +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import javax.annotation.Resource; +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class UpdateAccountServletTest extends BaseTest { + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Resource + private UpdateAccountServlet updateAccountServlet; + + private String getParam() { + return "{" + + " \"owner_address\": \"TXmVpin5vq5gdZsciyyjdZgKRUju4st1wM\"," + + " \"owner\": {" + + " \"type\": 0," + + " \"permission_name\": \"owner\"," + + " \"threshold\": 1," + + " \"keys\": [{" + + " \"address\": \"TXmVpin5vq5gdZsciyyjdZgKRUju4st1wM\"," + + " \"weight\": 1" + + " }]" + + " }," + + " \"witness\": {" + + " \"type\": 1," + + " \"permission_name\": \"witness\"," + + " \"threshold\": 1," + + " \"keys\": [{" + + " \"address\": \"TXmVpin5vq5gdZsciyyjdZgKRUju4st1wM\"," + + " \"weight\": 1" + + " }]" + + " }," + + " \"actives\": [{" + + " \"type\": 2," + + " \"permission_name\": \"active12323\"," + + " \"threshold\": 2," + + " \"operations\": " + + "\"7fff1fc0033e0000000000000000000000000000000000000000000000000000\"," + + " \"keys\": [{" + + " \"address\": \"TXmVpin5vq5gdZsciyyjdZgKRUju4st1wM\"," + + " \"weight\": 1" + + " }, {" + + " \"address\": \"TXmVpin5vq5gdZsciyyjdZgKRUju4st1wM\"," + + " \"weight\": 1" + + " }]" + + " }]," + + " \"visible\": true}"; + } + + @Test + public void test() { + String jsonParam = getParam(); + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes()); + MockHttpServletResponse response = new MockHttpServletResponse(); + + updateAccountServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + } +} From f9bab7a39b496d52dcfcdbc694d398f7e7db71b1 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Thu, 11 Jan 2024 17:32:09 +0800 Subject: [PATCH 33/61] feat(log): catch ContractExeException in createShieldedContractParameter (#5660) * feat(log): catch ContractExeException in createShieldedContractParameters --- .../src/main/java/org/tron/core/db/Manager.java | 1 + .../java/org/tron/core/services/RpcApiService.java | 3 ++- .../org/tron/core/capsule/utils/RLPListTest.java | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index b0d902eb84a..00af042beff 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -1843,6 +1843,7 @@ private void postSolidityLogContractTrigger(Long blockNum, Long lastSolidityNum) triggerCapsule.setTriggerName(Trigger.SOLIDITYLOG_TRIGGER_NAME); EventPluginLoader.getInstance().postSolidityLogTrigger(triggerCapsule); } else { + // when switch fork, block will be post to triggerCapsuleQueue, transaction may be not found logger.error("PostSolidityLogContractTrigger txId = {} not contains transaction.", triggerCapsule.getTransactionId()); } diff --git a/framework/src/main/java/org/tron/core/services/RpcApiService.java b/framework/src/main/java/org/tron/core/services/RpcApiService.java index c5077facf6f..85ad2dd18dd 100755 --- a/framework/src/main/java/org/tron/core/services/RpcApiService.java +++ b/framework/src/main/java/org/tron/core/services/RpcApiService.java @@ -89,6 +89,7 @@ import org.tron.core.config.args.Args; import org.tron.core.db.Manager; import org.tron.core.exception.BadItemException; +import org.tron.core.exception.ContractExeException; import org.tron.core.exception.ContractValidateException; import org.tron.core.exception.ItemNotFoundException; import org.tron.core.exception.NonUniqueObjectException; @@ -2475,7 +2476,7 @@ public void createShieldedContractParameters( ShieldedTRC20Parameters shieldedTRC20Parameters = wallet .createShieldedContractParameters(request); responseObserver.onNext(shieldedTRC20Parameters); - } catch (ZksnarkException | ContractValidateException e) { + } catch (ZksnarkException | ContractValidateException | ContractExeException e) { responseObserver.onError(getRunTimeException(e)); logger.info("createShieldedContractParameters: {}", e.getMessage()); return; diff --git a/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java b/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java index 7d903591966..160ecd8f6dd 100644 --- a/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java +++ b/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java @@ -1,5 +1,7 @@ package org.tron.core.capsule.utils; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import org.junit.Assert; import org.junit.Test; @@ -23,4 +25,14 @@ public void testGetRLPData() { Assert.assertEquals(new String(rlpList.getRLPData()), "rlpData"); } + @Test + public void testToBytes() + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Method method = RLP.class.getDeclaredMethod("toBytes", Object.class); + method.setAccessible(true); + + byte[] aBytes = new byte[10]; + byte[] bBytes = (byte[]) method.invoke(RLP.class, aBytes); + Assert.assertArrayEquals(aBytes, bBytes); + } } From 32f210dc7a0e94ca580c94af03d4c44dd209221f Mon Sep 17 00:00:00 2001 From: liukai Date: Thu, 11 Jan 2024 17:33:33 +0800 Subject: [PATCH 34/61] test(case): add transaction test cases --- .../org/tron/core/db/BlockIndexStoreTest.java | 64 +++++++++++++++++++ ...tTransactionInfoByBlockNumServletTest.java | 40 ++++++++++++ .../GetTransactionInfoByIdServletTest.java | 42 ++++++++++++ ...TransactionListFromPendingServletTest.java | 38 +++++++++++ 4 files changed, 184 insertions(+) create mode 100644 framework/src/test/java/org/tron/core/db/BlockIndexStoreTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByBlockNumServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByIdServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/GetTransactionListFromPendingServletTest.java diff --git a/framework/src/test/java/org/tron/core/db/BlockIndexStoreTest.java b/framework/src/test/java/org/tron/core/db/BlockIndexStoreTest.java new file mode 100644 index 00000000000..a5600b34b26 --- /dev/null +++ b/framework/src/test/java/org/tron/core/db/BlockIndexStoreTest.java @@ -0,0 +1,64 @@ +package org.tron.core.db; + +import com.google.protobuf.ByteString; +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.Constant; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.capsule.BytesCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.exception.ItemNotFoundException; + +public class BlockIndexStoreTest extends BaseTest { + + @Resource + private BlockIndexStore blockIndexStore; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath() + }, + Constant.TEST_CONF + ); + } + + private BlockCapsule getBlockCapsule(long number) { + return new BlockCapsule(number, Sha256Hash.ZERO_HASH, + System.currentTimeMillis(), ByteString.EMPTY); + } + + @Test + public void testPut() { + BlockCapsule blockCapsule = getBlockCapsule(1); + blockIndexStore.put(blockCapsule.getBlockId()); + byte[] key = ByteArray.fromLong(blockCapsule.getBlockId().getNum()); + Assert.assertTrue(blockIndexStore.has(key)); + } + + @Test + public void testGet() throws ItemNotFoundException { + BlockCapsule blockCapsule = getBlockCapsule(1); + blockIndexStore.put(blockCapsule.getBlockId()); + byte[] key = ByteArray.fromLong(blockCapsule.getBlockId().getNum()); + BytesCapsule bytesCapsule = blockIndexStore.get(key); + Assert.assertNotNull(bytesCapsule); + } + + @Test + public void testDelete() throws ItemNotFoundException { + BlockCapsule blockCapsule = getBlockCapsule(1); + blockIndexStore.put(blockCapsule.getBlockId()); + byte[] key = ByteArray.fromLong(blockCapsule.getBlockId().getNum()); + BytesCapsule bytesCapsule = blockIndexStore.get(key); + Assert.assertNotNull(bytesCapsule); + + blockIndexStore.delete(key); + BytesCapsule capsule = blockIndexStore.getUnchecked(key); + Assert.assertNull(capsule.getData()); + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByBlockNumServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByBlockNumServletTest.java new file mode 100644 index 00000000000..5dde8b4abce --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByBlockNumServletTest.java @@ -0,0 +1,40 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import javax.annotation.Resource; +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class GetTransactionInfoByBlockNumServletTest extends BaseTest { + + @Resource + private GetTransactionInfoByBlockNumServlet getTransactionInfoByBlockNumServlet; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Test + public void testGetTransactionInfoByBlockNum() { + String jsonParam = "{\"num\" : 100}"; + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + + getTransactionInfoByBlockNumServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByIdServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByIdServletTest.java new file mode 100644 index 00000000000..f2ff962c5a6 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByIdServletTest.java @@ -0,0 +1,42 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import javax.annotation.Resource; +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class GetTransactionInfoByIdServletTest extends BaseTest { + + @Resource + private GetTransactionInfoByIdServlet getTransactionInfoByIdServlet; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Test + public void testGetInfoById() { + String jsonParam = "{\"value\" : " + + "\"309b6fa3d01353e46f57dd8a8f276" + + "11f98e392b50d035cef213f2c55225a8bd2\"}"; + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes(UTF_8)); + MockHttpServletResponse response = new MockHttpServletResponse(); + + getTransactionInfoByIdServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + } +} diff --git a/framework/src/test/java/org/tron/core/services/http/GetTransactionListFromPendingServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetTransactionListFromPendingServletTest.java new file mode 100644 index 00000000000..614d520280d --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/GetTransactionListFromPendingServletTest.java @@ -0,0 +1,38 @@ +package org.tron.core.services.http; + +import static org.junit.Assert.assertEquals; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import javax.annotation.Resource; + +import org.apache.http.client.methods.HttpGet; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + + +public class GetTransactionListFromPendingServletTest extends BaseTest { + + @Resource + private GetTransactionListFromPendingServlet getTransactionListFromPendingServlet; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Test + public void testGet() { + MockHttpServletRequest request = createRequest(HttpGet.METHOD_NAME); + MockHttpServletResponse response = new MockHttpServletResponse(); + getTransactionListFromPendingServlet.doPost(request, response); + assertEquals(200, response.getStatus()); + } + +} From 5e7a40571f728b69667d9fb3bcc6aba656f0575c Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Fri, 12 Jan 2024 19:47:38 +0800 Subject: [PATCH 35/61] Merge master into release_v4.7.4 branch (#5669) * feat(db):optimize cache settings (#5659) * feat(lite):optimize DbLite tool (#5658) --- .../org/tron/core/store/WitnessStore.java | 22 +---- .../org/tron/consensus/ConsensusDelegate.java | 4 - .../consensus/dpos/MaintenanceManager.java | 4 +- .../main/java/org/tron/program/Version.java | 6 +- .../java/org/tron/core/db/ManagerTest.java | 17 +++- .../main/java/org/tron/plugins/DbLite.java | 93 ++++++++++++++++--- 6 files changed, 98 insertions(+), 48 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/store/WitnessStore.java b/chainbase/src/main/java/org/tron/core/store/WitnessStore.java index e01680cfc74..d23a73f92f9 100644 --- a/chainbase/src/main/java/org/tron/core/store/WitnessStore.java +++ b/chainbase/src/main/java/org/tron/core/store/WitnessStore.java @@ -11,10 +11,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import org.tron.common.cache.CacheManager; -import org.tron.common.cache.CacheStrategies; -import org.tron.common.cache.CacheType; -import org.tron.common.cache.TronCache; import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.Parameter; import org.tron.core.db.TronStoreWithRevoking; @@ -22,14 +18,10 @@ @Slf4j(topic = "DB") @Component public class WitnessStore extends TronStoreWithRevoking { - // cache for 127 SR - private final TronCache> witnessStandbyCache; @Autowired protected WitnessStore(@Value("witness") String dbName) { super(dbName); - String strategy = String.format(CacheStrategies.PATTERNS, 1, 1, "30s", 1); - witnessStandbyCache = CacheManager.allocate(CacheType.witnessStandby, strategy); } /** @@ -48,19 +40,8 @@ public WitnessCapsule get(byte[] key) { } public List getWitnessStandby() { - List list = - witnessStandbyCache.getIfPresent(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH); - if (list != null) { - return list; - } - return updateWitnessStandby(null); - } - - public List updateWitnessStandby(List all) { List ret; - if (all == null) { - all = getAllWitnesses(); - } + List all = getAllWitnesses(); all.sort(Comparator.comparingLong(WitnessCapsule::getVoteCount) .reversed().thenComparing(Comparator.comparingInt( (WitnessCapsule w) -> w.getAddress().hashCode()).reversed())); @@ -71,7 +52,6 @@ public List updateWitnessStandby(List all) { } // trim voteCount = 0 ret.removeIf(w -> w.getVoteCount() < 1); - witnessStandbyCache.put(Parameter.ChainConstant.WITNESS_STANDBY_LENGTH, ret); return ret; } diff --git a/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java b/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java index 4a98c933bd1..767463a6a5b 100644 --- a/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java +++ b/consensus/src/main/java/org/tron/consensus/ConsensusDelegate.java @@ -108,10 +108,6 @@ public List getAllWitnesses() { return witnessStore.getAllWitnesses(); } - public List updateWitnessStandby(List all) { - return witnessStore.updateWitnessStandby(all); - } - public void saveStateFlag(int flag) { dynamicPropertiesStore.saveStateFlag(flag); } diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index fc6cdd55c15..012169bdb87 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -151,13 +151,11 @@ public void doMaintenance() { if (dynamicPropertiesStore.allowChangeDelegation()) { long nextCycle = dynamicPropertiesStore.getCurrentCycleNumber() + 1; dynamicPropertiesStore.saveCurrentCycleNumber(nextCycle); - List all = consensusDelegate.getAllWitnesses(); - all.forEach(witness -> { + consensusDelegate.getAllWitnesses().forEach(witness -> { delegationStore.setBrokerage(nextCycle, witness.createDbKey(), delegationStore.getBrokerage(witness.createDbKey())); delegationStore.setWitnessVote(nextCycle, witness.createDbKey(), witness.getVoteCount()); }); - consensusDelegate.updateWitnessStandby(all); } } diff --git a/framework/src/main/java/org/tron/program/Version.java b/framework/src/main/java/org/tron/program/Version.java index 4740daa9d5e..a01eb714297 100644 --- a/framework/src/main/java/org/tron/program/Version.java +++ b/framework/src/main/java/org/tron/program/Version.java @@ -2,9 +2,9 @@ public class Version { - public static final String VERSION_NAME = "GreatVoyage-v4.7.2-140-g9d13f9cb69"; - public static final String VERSION_CODE = "18173"; - private static final String VERSION = "4.7.3"; + public static final String VERSION_NAME = "GreatVoyage-v4.7.3-5-g788136ebe"; + public static final String VERSION_CODE = "18180"; + private static final String VERSION = "4.7.3.1"; public static String getVersion() { return VERSION; diff --git a/framework/src/test/java/org/tron/core/db/ManagerTest.java b/framework/src/test/java/org/tron/core/db/ManagerTest.java index e25faf536f7..bd79f7d9776 100755 --- a/framework/src/test/java/org/tron/core/db/ManagerTest.java +++ b/framework/src/test/java/org/tron/core/db/ManagerTest.java @@ -581,14 +581,24 @@ public void pushSwitchFork() AccountResourceInsufficientException, EventBloomException { String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62"; + String key2 = "c85ef7d79691fe79573b1a7064c19c1a9819ebdbd1faaab1a8ec92344438aaf4"; byte[] privateKey = ByteArray.fromHexString(key); final ECKey ecKey = ECKey.fromPrivate(privateKey); byte[] address = ecKey.getAddress(); - + WitnessCapsule sr1 = new WitnessCapsule( + ByteString.copyFrom(address), "www.tron.net/first"); + sr1.setVoteCount(1000000000L); + byte[] privateKey2 = ByteArray.fromHexString(key2); + final ECKey ecKey2 = ECKey.fromPrivate(privateKey2); + byte[] address2 = ecKey2.getAddress(); + WitnessCapsule sr2 = new WitnessCapsule( + ByteString.copyFrom(address2), "www.tron.net/second"); + sr2.setVoteCount(100000L); + chainManager.getWitnessStore().put(address, sr1); WitnessCapsule witnessCapsule = new WitnessCapsule(ByteString.copyFrom(address)); chainManager.getWitnessScheduleStore().saveActiveWitnesses(new ArrayList<>()); chainManager.addWitness(ByteString.copyFrom(address)); - + List witnessStandby1 = chainManager.getWitnessStore().getWitnessStandby(); Block block = getSignedBlock(witnessCapsule.getAddress(), 1533529947843L, privateKey); dbManager.pushBlock(new BlockCapsule(block)); @@ -625,6 +635,9 @@ public void pushSwitchFork() } catch (Exception e) { Assert.assertTrue(e instanceof Exception); } + chainManager.getWitnessStore().put(address, sr2); + List witnessStandby2 = chainManager.getWitnessStore().getWitnessStandby(); + Assert.assertNotEquals(witnessStandby1, witnessStandby2); } diff --git a/plugins/src/main/java/org/tron/plugins/DbLite.java b/plugins/src/main/java/org/tron/plugins/DbLite.java index 8804d6210b2..732d4913021 100644 --- a/plugins/src/main/java/org/tron/plugins/DbLite.java +++ b/plugins/src/main/java/org/tron/plugins/DbLite.java @@ -3,6 +3,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import java.io.File; import java.io.FileNotFoundException; @@ -153,10 +155,10 @@ public void generateSnapshot(String sourceDir, String snapshotDir) { long start = System.currentTimeMillis(); snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString(); try { - mergeCheckpoint(sourceDir); hasEnoughBlock(sourceDir); List snapshotDbs = getSnapshotDbs(sourceDir); split(sourceDir, snapshotDir, snapshotDbs); + mergeCheckpoint2Snapshot(sourceDir, snapshotDir); // write genesisBlock , latest recent blocks and trans fillSnapshotBlockAndTransDb(sourceDir, snapshotDir); // save min block to info @@ -190,9 +192,9 @@ public void generateHistory(String sourceDir, String historyDir) { throw new IllegalStateException( String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir)); } - mergeCheckpoint(sourceDir); hasEnoughBlock(sourceDir); split(sourceDir, historyDir, archiveDbs); + mergeCheckpoint2History(sourceDir, historyDir); // save max block to info generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(), getLatestBlockHeaderNum(sourceDir)); @@ -261,6 +263,15 @@ private List getSnapshotDbs(String sourceDir) { return snapshotDbs; } + private void mergeCheckpoint2Snapshot(String sourceDir, String historyDir) { + List snapshotDbs = getSnapshotDbs(sourceDir); + mergeCheckpoint(sourceDir, historyDir, snapshotDbs); + } + + private void mergeCheckpoint2History(String sourceDir, String destDir) { + mergeCheckpoint(sourceDir, destDir, archiveDbs); + } + private void split(String sourceDir, String destDir, List dbs) throws IOException { logger.info("Begin to split the dbs."); spec.commandLine().getOut().println("Begin to split the dbs."); @@ -278,7 +289,7 @@ private void split(String sourceDir, String destDir, List dbs) throws IO FileUtils.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs); } - private void mergeCheckpoint(String sourceDir) { + private void mergeCheckpoint(String sourceDir, String destDir, List destDbs) { logger.info("Begin to merge checkpoint to dataset."); spec.commandLine().getOut().println("Begin to merge checkpoint to dataset."); try { @@ -287,18 +298,18 @@ private void mergeCheckpoint(String sourceDir) { for (String cp : cpList) { DBInterface checkpointDb = DbTool.getDB( sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, cp); - recover(checkpointDb, sourceDir); + recover(checkpointDb, destDir, destDbs); } } else if (Paths.get(sourceDir, CHECKPOINT_DB).toFile().exists()) { DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); - recover(tmpDb, sourceDir); + recover(tmpDb, destDir, destDbs); } } catch (IOException | RocksDBException e) { throw new RuntimeException(e); } } - private void recover(DBInterface db, String destDir) + private void recover(DBInterface db, String destDir, List destDbs) throws IOException, RocksDBException { try (DBIterator iterator = db.iterator()) { for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { @@ -312,15 +323,17 @@ private void recover(DBInterface db, String destDir) byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length); byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); - DBInterface destDb = DbTool.getDB(destDir, dbName); - if (realValue != null) { - destDb.put(realKey, realValue); - } else { - byte op = value[0]; - if (DBUtils.Operator.DELETE.getValue() == op) { - destDb.delete(realKey); + if (destDbs != null && destDbs.contains(dbName)) { + DBInterface destDb = DbTool.getDB(destDir, dbName); + if (realValue != null) { + destDb.put(realKey, realValue); } else { - destDb.put(realKey, new byte[0]); + byte op = value[0]; + if (DBUtils.Operator.DELETE.getValue() == op) { + destDb.delete(realKey); + } else { + destDb.put(realKey, new byte[0]); + } } } } @@ -340,7 +353,15 @@ private void generateInfoProperties(String propertyfile, long num) } private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException { + // query latest_block_header_number from checkpoint first final String latestBlockHeaderNumber = "latest_block_header_number"; + DBInterface checkpointDb = getCheckpointDb(databaseDir); + Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb, + latestBlockHeaderNumber.getBytes()); + if (blockNumber != null) { + return blockNumber; + } + // query from propertiesDb if checkpoint not contains latest_block_header_number DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME); return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber))) .map(ByteArray::toLong) @@ -348,6 +369,14 @@ private long getLatestBlockHeaderNum(String databaseDir) throws IOException, Roc () -> new IllegalArgumentException("not found latest block header number")); } + private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) { + byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key)); + if (value != null && value.length > 1) { + return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length)); + } + return null; + } + /** * recent blocks, trans and genesis block. */ @@ -414,6 +443,15 @@ private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDB return result; } + private static byte[] simpleEncode(String s) { + byte[] bytes = s.getBytes(); + byte[] length = Ints.toByteArray(bytes.length); + byte[] r = new byte[4 + bytes.length]; + System.arraycopy(length, 0, r, 0, 4); + System.arraycopy(bytes, 0, r, 4, bytes.length); + return r; + } + private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String liteDir) throws IOException, RocksDBException { logger.info("Check the compatibility of this history."); @@ -485,6 +523,7 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo) DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME); DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME); + ProgressBar.wrap(LongStream.rangeClosed(start, end) .boxed() .sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> { @@ -519,6 +558,7 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws return; } + Path bakDir = Paths.get(liteDir, BACKUP_DIR_PREFIX + START_TIME); logger.info("Begin to merge {} to database, start {} end {}.", bakDir, start, end); spec.commandLine().getOut() @@ -545,7 +585,17 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key) throws IOException, RocksDBException { - byte[] value = DbTool.getDB(sourceDir, dbName).get(key); + DBInterface sourceDb = DbTool.getDB(sourceDir, dbName); + DBInterface checkpointDb = getCheckpointDb(sourceDir); + // get data from tmp first. + byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key)); + byte[] value; + if (isEmptyBytes(valueFromTmp)) { + value = sourceDb.get(key); + } else { + value = valueFromTmp.length == 1 + ? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length); + } if (isEmptyBytes(value)) { throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s", dbName, Arrays.toString(key))); @@ -614,6 +664,19 @@ private long getSecondBlock(String databaseDir) throws RocksDBException, IOExcep return num; } + private DBInterface getCheckpointDb(String sourceDir) throws IOException, RocksDBException { + List cpList = getCheckpointV2List(sourceDir); + DBInterface checkpointDb; + if (cpList.size() > 0) { + String latestCp = cpList.get(cpList.size() - 1); + checkpointDb = DbTool.getDB( + sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, latestCp); + } else { + checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); + } + return checkpointDb; + } + @VisibleForTesting public static void setRecentBlks(long recentBlks) { RECENT_BLKS = recentBlks; From 32e1e151a4c9ab748f3f31bd5a25a0b6c4db4909 Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Mon, 15 Jan 2024 11:39:49 +0800 Subject: [PATCH 36/61] feat(db): optimize calculation services for reward when the node starts (#5654) * feat(db): optimize calculation services for reward when the service starts * feat(log): change log from error to warn for merkle root mismatch --- .../tron/core/service/RewardViCalService.java | 16 +++++++++++----- .../src/main/java/org/tron/core/db/Manager.java | 5 +++++ .../tron/core/services/ComputeRewardTest.java | 4 ---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java b/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java index acb16142b3f..33e5233b551 100644 --- a/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java +++ b/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java @@ -17,7 +17,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.LongStream; -import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -75,8 +74,14 @@ public RewardViCalService(@Autowired DynamicPropertiesStore propertiesStore, this.witnessStore = witnessStore.getDb(); } - @PostConstruct - private void init() { + public void init() { + // after init, we can get the latest block header number from db + this.newRewardCalStartCycle = this.getNewRewardAlgorithmEffectiveCycle(); + boolean ret = this.newRewardCalStartCycle != Long.MAX_VALUE; + if (ret) { + // checkpoint is flushed to db, we can start rewardViCalService immediately + lastBlockNumber = Long.MAX_VALUE; + } es.scheduleWithFixedDelay(this::maybeRun, 0, 3, TimeUnit.SECONDS); } @@ -100,7 +105,8 @@ private void maybeRun() { this.clearUp(true); logger.info("rewardViCalService is already done"); } else { - if (this.getLatestBlockHeaderNumber() > lastBlockNumber) { + if (lastBlockNumber == Long.MAX_VALUE // start rewardViCalService immediately + || this.getLatestBlockHeaderNumber() > lastBlockNumber) { // checkpoint is flushed to db, so we can start rewardViCalService startRewardCal(); clearUp(true); @@ -170,7 +176,7 @@ private void calcMerkleRoot() { Sha256Hash rewardViRootLocal = MerkleTree.getInstance().createTree(ids).getRoot().getHash(); if (!Objects.equals(rewardViRoot, rewardViRootLocal)) { - logger.error("merkle root mismatch, expect: {}, actual: {}", + logger.warn("merkle root mismatch, expect: {}, actual: {}", rewardViRoot, rewardViRootLocal); } logger.info("calcMerkleRoot: {}", rewardViRootLocal); diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 00af042beff..445c48fdd94 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -130,6 +130,7 @@ import org.tron.core.metrics.MetricsKey; import org.tron.core.metrics.MetricsUtil; import org.tron.core.service.MortgageService; +import org.tron.core.service.RewardViCalService; import org.tron.core.store.AccountAssetStore; import org.tron.core.store.AccountIdIndexStore; import org.tron.core.store.AccountIndexStore; @@ -260,6 +261,9 @@ public class Manager { private ExecutorService filterEs; private static final String filterEsName = "filter"; + @Autowired + private RewardViCalService rewardViCalService; + /** * Cycle thread to rePush Transactions */ @@ -465,6 +469,7 @@ public void init() { revokingStore.disable(); revokingStore.check(); transactionCache.initCache(); + rewardViCalService.init(); this.setProposalController(ProposalController.createInstance(this)); this.setMerkleContainer( merkleContainer.createInstance(chainBaseManager.getMerkleTreeStore(), diff --git a/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java b/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java index 17b2526d6d0..00e0799cf11 100644 --- a/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java +++ b/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java @@ -22,7 +22,6 @@ import org.tron.common.error.TronDBException; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.utils.ByteArray; -import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.WitnessCapsule; @@ -251,9 +250,6 @@ private void setUp() { } catch (ExecutionException e) { throw new TronDBException(e); } - - rewardViCalService.setRewardViRoot(Sha256Hash.wrap( - ByteString.fromHex("e0ebe2f3243391ed674dff816a07f589a3279420d6d88bc823b6a9d5778337ce"))); } @Test From 46454a24824bdb74a3d6063ef119cf2a8300eed9 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Mon, 15 Jan 2024 12:39:42 +0800 Subject: [PATCH 37/61] feat(log):verify columns of hellomessage to avoid too long log (#5667) --- .../net/message/handshake/HelloMessage.java | 17 ++++++++ .../service/handshake/HandshakeService.java | 15 ++++--- .../net/services/HandShakeServiceTest.java | 41 +++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java b/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java index b98d0e173fb..867ced5dbff 100755 --- a/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java @@ -4,6 +4,7 @@ import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.tron.common.utils.ByteArray; +import org.tron.common.utils.DecodeUtil; import org.tron.common.utils.StringUtil; import org.tron.core.ChainBaseManager; import org.tron.core.capsule.BlockCapsule; @@ -169,6 +170,22 @@ public boolean valid() { return false; } + int maxByteSize = 200; + ByteString address = this.helloMessage.getAddress(); + if (!address.isEmpty() && address.toByteArray().length > maxByteSize) { + return false; + } + + ByteString sig = this.helloMessage.getSignature(); + if (!sig.isEmpty() && sig.toByteArray().length > maxByteSize) { + return false; + } + + ByteString codeVersion = this.helloMessage.getCodeVersion(); + if (!codeVersion.isEmpty() && codeVersion.toByteArray().length > maxByteSize) { + return false; + } + return true; } diff --git a/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java b/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java index aa6567d1cda..6cd117c83dd 100644 --- a/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java +++ b/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java @@ -48,12 +48,15 @@ public void processHelloMessage(PeerConnection peer, HelloMessage msg) { } if (!msg.valid()) { - logger.warn("Peer {} invalid hello message parameters, " - + "GenesisBlockId: {}, SolidBlockId: {}, HeadBlockId: {}", - peer.getInetSocketAddress(), - ByteArray.toHexString(msg.getInstance().getGenesisBlockId().getHash().toByteArray()), - ByteArray.toHexString(msg.getInstance().getSolidBlockId().getHash().toByteArray()), - ByteArray.toHexString(msg.getInstance().getHeadBlockId().getHash().toByteArray())); + logger.warn("Peer {} invalid hello message parameters, GenesisBlockId: {}, SolidBlockId: {}, " + + "HeadBlockId: {}, address: {}, sig: {}, codeVersion: {}", + peer.getInetSocketAddress(), + ByteArray.toHexString(msg.getInstance().getGenesisBlockId().getHash().toByteArray()), + ByteArray.toHexString(msg.getInstance().getSolidBlockId().getHash().toByteArray()), + ByteArray.toHexString(msg.getInstance().getHeadBlockId().getHash().toByteArray()), + msg.getInstance().getAddress().toByteArray().length, + msg.getInstance().getSignature().toByteArray().length, + msg.getInstance().getCodeVersion().toByteArray().length); peer.disconnect(ReasonCode.UNEXPECTED_IDENTITY); return; } diff --git a/framework/src/test/java/org/tron/core/net/services/HandShakeServiceTest.java b/framework/src/test/java/org/tron/core/net/services/HandShakeServiceTest.java index ea2156ae713..f4fabce5d64 100644 --- a/framework/src/test/java/org/tron/core/net/services/HandShakeServiceTest.java +++ b/framework/src/test/java/org/tron/core/net/services/HandShakeServiceTest.java @@ -21,6 +21,7 @@ import org.mockito.Mockito; import org.springframework.context.ApplicationContext; import org.tron.common.application.TronApplicationContext; +import org.tron.common.utils.ByteArray; import org.tron.common.utils.ReflectUtils; import org.tron.common.utils.Sha256Hash; import org.tron.core.ChainBaseManager; @@ -137,6 +138,37 @@ public void testInvalidHelloMessage() { } } + @Test + public void testInvalidHelloMessage2() throws Exception { + Protocol.HelloMessage.Builder builder = getTestHelloMessageBuilder(); + Assert.assertTrue(new HelloMessage(builder.build().toByteArray()).valid()); + + builder.setAddress(ByteString.copyFrom(new byte[201])); + HelloMessage helloMessage = new HelloMessage(builder.build().toByteArray()); + Assert.assertFalse(helloMessage.valid()); + + builder.setAddress(ByteString.copyFrom(new byte[200])); + helloMessage = new HelloMessage(builder.build().toByteArray()); + Assert.assertTrue(helloMessage.valid()); + + builder.setSignature(ByteString.copyFrom(new byte[201])); + helloMessage = new HelloMessage(builder.build().toByteArray()); + Assert.assertFalse(helloMessage.valid()); + + builder.setSignature(ByteString.copyFrom(new byte[200])); + helloMessage = new HelloMessage(builder.build().toByteArray()); + Assert.assertTrue(helloMessage.valid()); + + builder.setCodeVersion(ByteString.copyFrom(new byte[201])); + helloMessage = new HelloMessage(builder.build().toByteArray()); + Assert.assertFalse(helloMessage.valid()); + + builder.setCodeVersion(ByteString.copyFrom(new byte[200])); + helloMessage = new HelloMessage(builder.build().toByteArray()); + Assert.assertTrue(helloMessage.valid()); + } + + @Test public void testRelayHelloMessage() throws NoSuchMethodException { InetSocketAddress a1 = new InetSocketAddress("127.0.0.1", 10001); @@ -266,4 +298,13 @@ private Protocol.HelloMessage.Builder getHelloMessageBuilder(Node from, long tim return builder; } + + private Protocol.HelloMessage.Builder getTestHelloMessageBuilder() { + InetSocketAddress a1 = new InetSocketAddress("127.0.0.1", 10001); + Node node = new Node(NetUtil.getNodeId(), a1.getAddress().getHostAddress(), null, a1.getPort()); + Protocol.HelloMessage.Builder builder = + getHelloMessageBuilder(node, System.currentTimeMillis(), + ChainBaseManager.getChainBaseManager()); + return builder; + } } From 1225ace361bfce61f61f93305f3b407c89dfd9f1 Mon Sep 17 00:00:00 2001 From: liukai Date: Mon, 15 Jan 2024 20:25:37 +0800 Subject: [PATCH 38/61] test(case): optimize case --- .../core/actuator/ActuatorFactoryTest.java | 71 +++++++++++++++ .../AccountPermissionUpdateServletTest.java | 88 ------------------- .../services/http/ClearABIServletTest.java | 55 +++++++++++- .../http/CreateAccountServletTest.java | 33 ++++++- .../http/CreateAssetIssueServletTest.java | 33 ++++++- .../http/CreateSpendAuthSigServletTest.java | 60 +++++++++++++ .../http/CreateWitnessServletTest.java | 88 +++++++++++++++++++ ...tTransactionInfoByBlockNumServletTest.java | 39 ++++++++ .../GetTransactionInfoByIdServletTest.java | 88 ++++++++++++++++++- 9 files changed, 457 insertions(+), 98 deletions(-) create mode 100644 framework/src/test/java/org/tron/core/actuator/ActuatorFactoryTest.java delete mode 100644 framework/src/test/java/org/tron/core/services/http/AccountPermissionUpdateServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/CreateSpendAuthSigServletTest.java create mode 100644 framework/src/test/java/org/tron/core/services/http/CreateWitnessServletTest.java diff --git a/framework/src/test/java/org/tron/core/actuator/ActuatorFactoryTest.java b/framework/src/test/java/org/tron/core/actuator/ActuatorFactoryTest.java new file mode 100644 index 00000000000..8258fbf9a3e --- /dev/null +++ b/framework/src/test/java/org/tron/core/actuator/ActuatorFactoryTest.java @@ -0,0 +1,71 @@ +package org.tron.core.actuator; + +import com.google.protobuf.ByteString; +import java.util.List; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; +import org.tron.core.Constant; +import org.tron.core.Wallet; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.TransactionCapsule; +import org.tron.core.config.args.Args; +import org.tron.protos.Protocol.AccountType; +import org.tron.protos.contract.BalanceContract.TransferContract; + +public class ActuatorFactoryTest extends BaseTest { + + private static final String OWNER_ADDRESS = Wallet.getAddressPreFixString() + + "548794500882809695a8a687866e76d4271a1abc"; + private static final String TO_ADDRESS = Wallet.getAddressPreFixString() + + "abd4b9367799eaa3197fecb144eb71de1e049abc"; + + static { + Args.setParam( + new String[] { + "--output-directory", dbPath() + }, + Constant.TEST_CONF + ); + } + + private TransferContract getContract(long count, String owneraddress, String toaddress) { + return TransferContract.newBuilder() + .setOwnerAddress(ByteString.copyFrom(ByteArray.fromHexString(owneraddress))) + .setToAddress(ByteString.copyFrom(ByteArray.fromHexString(toaddress))) + .setAmount(count) + .build(); + } + + @Before + public void createCapsule() { + AccountCapsule ownerCapsule = + new AccountCapsule( + ByteString.copyFromUtf8("owner"), + ByteString.copyFrom(ByteArray.fromHexString(OWNER_ADDRESS)), + AccountType.Normal, + 10000L); + AccountCapsule toAccountCapsule = + new AccountCapsule( + ByteString.copyFromUtf8("toAccount"), + ByteString.copyFrom(ByteArray.fromHexString(TO_ADDRESS)), + AccountType.Normal, + 10L); + dbManager.getAccountStore().put(ownerCapsule.getAddress().toByteArray(), ownerCapsule); + dbManager.getAccountStore().put(toAccountCapsule.getAddress().toByteArray(), toAccountCapsule); + } + + + @Test + public void testCreateActuator() { + TransferContract contract = getContract(10L, OWNER_ADDRESS, TO_ADDRESS); + TransactionCapsule trx = new TransactionCapsule(contract, + chainBaseManager.getAccountStore()); + List actList = ActuatorFactory.createActuator(trx, chainBaseManager); + + Assert.assertEquals(1, actList.size()); + } + +} diff --git a/framework/src/test/java/org/tron/core/services/http/AccountPermissionUpdateServletTest.java b/framework/src/test/java/org/tron/core/services/http/AccountPermissionUpdateServletTest.java deleted file mode 100644 index 313a0a12a60..00000000000 --- a/framework/src/test/java/org/tron/core/services/http/AccountPermissionUpdateServletTest.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.tron.core.services.http; - -import static org.tron.common.utils.client.utils.HttpMethed.createRequest; - -import javax.annotation.Resource; - -import org.apache.http.client.methods.HttpPost; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.mock.web.MockHttpServletRequest; -import org.springframework.mock.web.MockHttpServletResponse; -import org.tron.common.BaseTest; -import org.tron.core.Constant; -import org.tron.core.config.args.Args; - -public class AccountPermissionUpdateServletTest extends BaseTest { - - static { - Args.setParam( - new String[]{ - "--output-directory", dbPath(), - }, Constant.TEST_CONF - ); - } - - @Resource - private AccountPermissionUpdateServlet accountPermissionUpdateServlet; - - private String getParam() { - return "{" - + " \"owner_address\":\"TRGhNNfnmgLegT4zHNjEqDSADjgmnHvubJ\"," - + " \"owner\":{" - + " \"type\":0," - + " \"permission_name\":\"owner\"," - + " \"threshold\":1," - + " \"keys\":[" - + " {" - + " \"address\":\"TRGhNNfnmgLegT4zHNjEqDSADjgmnHvubJ\"," - + " \"weight\":1" - + " }" - + " ]" - + " }," - + " \"witness\":{" - + " \"type\":1," - + " \"permission_name\":\"witness\"," - + " \"threshold\":1," - + " \"keys\":[" - + " {" - + " \"address\":\"TRGhNNfnmgLegT4zHNjEqDSADjgmnHvubJ\"," - + " \"weight\":1" - + " }" - + " ]" - + " }," - + " \"actives\":[" - + " {" - + " \"type\":2," - + " \"permission_name\":\"active12323\"," - + " \"threshold\":2," - + " \"operations\":" - + "\"7fff1fc0033e0000000000000000000000000000000000000000000000000000\"," - + " \"keys\":[" - + " {" - + " \"address\":\"TNhXo1GbRNCuorvYu5JFWN3m2NYr9QQpVR\"," - + " \"weight\":1" - + " }," - + " {" - + " \"address\":\"TKwhcDup8L2PH5r6hxp5CQvQzZqJLmKvZP\"," - + " \"weight\":1" - + " }" - + " ]" - + " }" - + " ]," - + " \"visible\": true" - + "}"; - } - - @Test - public void test() { - String jsonParam = getParam(); - MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); - request.setContentType("application/json"); - request.setContent(jsonParam.getBytes()); - MockHttpServletResponse response = new MockHttpServletResponse(); - - accountPermissionUpdateServlet.doPost(request, response); - Assert.assertEquals(200, response.getStatus()); - } -} diff --git a/framework/src/test/java/org/tron/core/services/http/ClearABIServletTest.java b/framework/src/test/java/org/tron/core/services/http/ClearABIServletTest.java index 27bc2a6eb34..a3a051ec6c9 100644 --- a/framework/src/test/java/org/tron/core/services/http/ClearABIServletTest.java +++ b/framework/src/test/java/org/tron/core/services/http/ClearABIServletTest.java @@ -1,18 +1,28 @@ package org.tron.core.services.http; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.tron.common.utils.client.utils.HttpMethed.createRequest; +import com.alibaba.fastjson.JSONObject; +import com.google.protobuf.ByteString; + +import java.io.UnsupportedEncodingException; import javax.annotation.Resource; import org.apache.http.client.methods.HttpPost; + import org.junit.Assert; import org.junit.Test; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; import org.tron.core.Constant; +import org.tron.core.capsule.ContractCapsule; import org.tron.core.config.args.Args; +import org.tron.protos.contract.SmartContractOuterClass; public class ClearABIServletTest extends BaseTest { @@ -27,11 +37,39 @@ public class ClearABIServletTest extends BaseTest { @Resource private ClearABIServlet clearABIServlet; + private static final String SMART_CONTRACT_NAME = "smart_contract_test"; + private static String CONTRACT_ADDRESS = "A0B4750E2CD76E19DCA331BF5D089B71C3C2798548"; + private static String OWNER_ADDRESS; + private static final long SOURCE_ENERGY_LIMIT = 10L; + + + + private SmartContractOuterClass.SmartContract.Builder createContract( + String contractAddress, String contractName) { + OWNER_ADDRESS = + "A099357684BC659F5166046B56C95A0E99F1265CBD"; + SmartContractOuterClass.SmartContract.Builder builder = + SmartContractOuterClass.SmartContract.newBuilder(); + builder.setName(contractName); + builder.setOriginAddress(ByteString.copyFrom(ByteArray.fromHexString(OWNER_ADDRESS))); + builder.setContractAddress(ByteString.copyFrom(ByteArray.fromHexString(contractAddress))); + builder.setOriginEnergyLimit(SOURCE_ENERGY_LIMIT); + return builder; + } + @Test - public void testClear() { - String jsonParam = "{\n" - + " \"owner_address\": \"41a7d8a35b260395c14aa456297662092ba3b76fc0\",\n" - + " \"contract_address\": \"417bcb781f4743afaacf9f9528f3ea903b3782339f\"\n" + public void testClearABI() { + chainBaseManager.getDynamicPropertiesStore() + .saveAllowTvmConstantinople(1); + SmartContractOuterClass.SmartContract.Builder contract = + createContract(CONTRACT_ADDRESS, SMART_CONTRACT_NAME); + chainBaseManager.getContractStore().put( + ByteArray.fromHexString(CONTRACT_ADDRESS), + new ContractCapsule(contract.build())); + + String jsonParam = "{" + + " \"owner_address\": \"A099357684BC659F5166046B56C95A0E99F1265CBD\"," + + " \"contract_address\": \"A0B4750E2CD76E19DCA331BF5D089B71C3C2798548\"" + "}"; MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); request.setContentType("application/json"); @@ -40,6 +78,15 @@ public void testClear() { MockHttpServletResponse response = new MockHttpServletResponse(); clearABIServlet.doPost(request, response); Assert.assertEquals(200, response.getStatus()); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + assertTrue(result.containsKey("raw_data")); + assertTrue(result.containsKey("txID")); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } } diff --git a/framework/src/test/java/org/tron/core/services/http/CreateAccountServletTest.java b/framework/src/test/java/org/tron/core/services/http/CreateAccountServletTest.java index 70a319e6040..bbc00ce81f0 100644 --- a/framework/src/test/java/org/tron/core/services/http/CreateAccountServletTest.java +++ b/framework/src/test/java/org/tron/core/services/http/CreateAccountServletTest.java @@ -1,18 +1,27 @@ package org.tron.core.services.http; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; import static org.tron.common.utils.client.utils.HttpMethed.createRequest; +import com.alibaba.fastjson.JSONObject; +import com.google.protobuf.ByteString; + +import java.io.UnsupportedEncodingException; import javax.annotation.Resource; import org.apache.http.client.methods.HttpPost; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; import org.tron.core.Constant; +import org.tron.core.capsule.AccountCapsule; import org.tron.core.config.args.Args; +import org.tron.protos.Protocol; public class CreateAccountServletTest extends BaseTest { @@ -28,11 +37,23 @@ public class CreateAccountServletTest extends BaseTest { @Resource private CreateAccountServlet createAccountServlet; + @Before + public void init() { + AccountCapsule accountCapsule = new AccountCapsule( + ByteString.copyFrom(ByteArray + .fromHexString("A099357684BC659F5166046B56C95A0E99F1265CD1")), + ByteString.copyFromUtf8("owner"), + Protocol.AccountType.forNumber(1)); + + chainBaseManager.getAccountStore().put(accountCapsule.createDbKey(), + accountCapsule); + } + @Test public void testCreate() { String jsonParam = "{" - + "\"owner_address\": \"41d1e7a6bc354106cb410e65ff8b181c600ff14292\"," - + "\"account_address\": \"41e552f6487585c2b58bc2c9bb4492bc1f17132cd0\"" + + "\"owner_address\": \"A099357684BC659F5166046B56C95A0E99F1265CD1\"," + + "\"account_address\": \"A0B4750E2CD76E19DCA331BF5D089B71C3C2798541\"" + "}"; MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); request.setContentType("application/json"); @@ -42,5 +63,13 @@ public void testCreate() { createAccountServlet.doPost(request, response); Assert.assertEquals(200, response.getStatus()); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + Assert.assertTrue(result.containsKey("raw_data")); + Assert.assertTrue(result.containsKey("txID")); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } } } diff --git a/framework/src/test/java/org/tron/core/services/http/CreateAssetIssueServletTest.java b/framework/src/test/java/org/tron/core/services/http/CreateAssetIssueServletTest.java index d9f9ccc412c..4c70eb9252c 100644 --- a/framework/src/test/java/org/tron/core/services/http/CreateAssetIssueServletTest.java +++ b/framework/src/test/java/org/tron/core/services/http/CreateAssetIssueServletTest.java @@ -1,18 +1,27 @@ package org.tron.core.services.http; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; import static org.tron.common.utils.client.utils.HttpMethed.createRequest; +import com.alibaba.fastjson.JSONObject; +import com.google.protobuf.ByteString; + +import java.io.UnsupportedEncodingException; import javax.annotation.Resource; import org.apache.http.client.methods.HttpPost; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; import org.tron.core.Constant; +import org.tron.core.capsule.AccountCapsule; import org.tron.core.config.args.Args; +import org.tron.protos.Protocol; public class CreateAssetIssueServletTest extends BaseTest { @@ -24,14 +33,26 @@ public class CreateAssetIssueServletTest extends BaseTest { ); } - @Resource private CreateAssetIssueServlet createAssetIssueServlet; + @Before + public void init() { + AccountCapsule accountCapsule = new AccountCapsule( + ByteString.copyFrom(ByteArray + .fromHexString("A099357684BC659F5166046B56C95A0E99F1265CD1")), + ByteString.copyFromUtf8("owner"), + Protocol.AccountType.forNumber(1)); + accountCapsule.setBalance(10000000000L); + + chainBaseManager.getAccountStore().put(accountCapsule.createDbKey(), + accountCapsule); + } + @Test public void testCreate() { String jsonParam = "{" - + " \"owner_address\": \"41e552f6487585c2b58bc2c9bb4492bc1f17132cd0\"," + + " \"owner_address\": \"A099357684BC659F5166046B56C95A0E99F1265CD1\"," + " \"name\": \"0x6173736574497373756531353330383934333132313538\"," + " \"abbr\": \"0x6162627231353330383934333132313538\"," + " \"total_supply\": 4321," @@ -55,6 +76,14 @@ public void testCreate() { MockHttpServletResponse response = new MockHttpServletResponse(); createAssetIssueServlet.doPost(request, response); Assert.assertEquals(200, response.getStatus()); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + Assert.assertTrue(result.containsKey("raw_data")); + Assert.assertTrue(result.containsKey("txID")); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } } diff --git a/framework/src/test/java/org/tron/core/services/http/CreateSpendAuthSigServletTest.java b/framework/src/test/java/org/tron/core/services/http/CreateSpendAuthSigServletTest.java new file mode 100644 index 00000000000..301e4472e69 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/CreateSpendAuthSigServletTest.java @@ -0,0 +1,60 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import com.alibaba.fastjson.JSONObject; +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +public class CreateSpendAuthSigServletTest extends BaseTest { + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + @Resource + private CreateSpendAuthSigServlet createSpendAuthSigServlet; + + @Test + public void testCreateSpendAuthSig() { + String jsonParam = "{" + + " \"ask\": \"e3ebcba1531f6d9158d9c162660c5d7c04dadf77d" + + "85d7436a9c98b291ff69a09\"," + + " \"tx_hash\": \"3b78fee6e956f915ffe082284c5f18640edca9" + + "c57a5f227e5f7d7eb65ad61502\"," + + " \"alpha\": \"2608999c3a97d005a879ecdaa16fd29ae434fb67" + + "b177c5e875b0c829e6a1db04\"" + + "}"; + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes(UTF_8)); + + MockHttpServletResponse response = new MockHttpServletResponse(); + createSpendAuthSigServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + Assert.assertTrue(result.containsKey("value")); + String resultValue = (String) result.get("value"); + Assert.assertNotNull(resultValue); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + +} diff --git a/framework/src/test/java/org/tron/core/services/http/CreateWitnessServletTest.java b/framework/src/test/java/org/tron/core/services/http/CreateWitnessServletTest.java new file mode 100644 index 00000000000..6cd6e9e2482 --- /dev/null +++ b/framework/src/test/java/org/tron/core/services/http/CreateWitnessServletTest.java @@ -0,0 +1,88 @@ +package org.tron.core.services.http; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; +import static org.tron.common.utils.client.utils.HttpMethed.createRequest; + +import com.alibaba.fastjson.JSONObject; +import com.google.protobuf.ByteString; + +import java.io.UnsupportedEncodingException; +import javax.annotation.Resource; + +import org.apache.http.client.methods.HttpPost; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.tron.common.BaseTest; +import org.tron.common.crypto.ECKey; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Utils; +import org.tron.core.Constant; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.config.args.Args; +import org.tron.protos.Protocol; + +public class CreateWitnessServletTest extends BaseTest { + + @Resource + private CreateWitnessServlet createWitnessServlet; + + static { + Args.setParam( + new String[]{ + "--output-directory", dbPath(), + }, Constant.TEST_CONF + ); + } + + private static WitnessCapsule witnessCapsule; + private static AccountCapsule accountCapsule; + + @Before + public void init() { + ECKey ecKey = new ECKey(Utils.getRandom()); + ByteString address = ByteString.copyFrom(ecKey.getAddress()); + + accountCapsule = + new AccountCapsule(Protocol.Account + .newBuilder() + .setAddress(address).build()); + accountCapsule.setBalance(10000000L); + dbManager.getAccountStore().put(accountCapsule + .getAddress().toByteArray(), accountCapsule); + } + + @Test + public void testCreateWitness() { + chainBaseManager.getDynamicPropertiesStore() + .saveAccountUpgradeCost(1L); + String hexAddress = ByteArray + .toHexString(accountCapsule.getAddress().toByteArray()); + String jsonParam = "{\"owner_address\":\"" + + hexAddress + "\"," + + " \"url\": \"00757064617" + + "4654e616d6531353330363038383733343633\"}"; + MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); + request.setContentType("application/json"); + request.setContent(jsonParam.getBytes(UTF_8)); + + MockHttpServletResponse response = new MockHttpServletResponse(); + createWitnessServlet.doPost(request, response); + Assert.assertEquals(200, response.getStatus()); + try { + String contentAsString = response.getContentAsString(); + JSONObject result = JSONObject.parseObject(contentAsString); + Assert.assertTrue(result.containsKey("raw_data")); + Assert.assertTrue(result.containsKey("txID")); + } catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + } + } + +} + + diff --git a/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByBlockNumServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByBlockNumServletTest.java index 5dde8b4abce..0a1a2e4ac5a 100644 --- a/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByBlockNumServletTest.java +++ b/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByBlockNumServletTest.java @@ -3,20 +3,33 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.tron.common.utils.client.utils.HttpMethed.createRequest; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +import java.io.UnsupportedEncodingException; import javax.annotation.Resource; + import org.apache.http.client.methods.HttpPost; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; import org.tron.core.Constant; + +import org.tron.core.capsule.TransactionInfoCapsule; +import org.tron.core.capsule.TransactionRetCapsule; import org.tron.core.config.args.Args; +import org.tron.core.db.TransactionStoreTest; public class GetTransactionInfoByBlockNumServletTest extends BaseTest { @Resource private GetTransactionInfoByBlockNumServlet getTransactionInfoByBlockNumServlet; + private static final byte[] transactionId = TransactionStoreTest.randomBytes(32); + private static TransactionRetCapsule transactionRetCapsule; static { Args.setParam( @@ -26,6 +39,22 @@ public class GetTransactionInfoByBlockNumServletTest extends BaseTest { ); } + @Before + public void init() { + byte[] blockNum = ByteArray.fromLong(100); + TransactionInfoCapsule transactionInfoCapsule = new TransactionInfoCapsule(); + + transactionInfoCapsule.setId(transactionId); + transactionInfoCapsule.setFee(1000L); + transactionInfoCapsule.setBlockNumber(100L); + transactionInfoCapsule.setBlockTimeStamp(200L); + + transactionRetCapsule = new TransactionRetCapsule(); + transactionRetCapsule.addTransactionInfo(transactionInfoCapsule.getInstance()); + chainBaseManager.getTransactionRetStore() + .put(blockNum, transactionRetCapsule); + } + @Test public void testGetTransactionInfoByBlockNum() { String jsonParam = "{\"num\" : 100}"; @@ -36,5 +65,15 @@ public void testGetTransactionInfoByBlockNum() { getTransactionInfoByBlockNumServlet.doPost(request, response); Assert.assertEquals(200, response.getStatus()); + try { + String contentAsString = response.getContentAsString(); + JSONArray array = JSONArray.parseArray(contentAsString); + Assert.assertEquals(1, array.size()); + JSONObject object = (JSONObject) array.get(0); + Assert.assertEquals(1000, object.get("fee")); + Assert.assertEquals(100, object.get("blockNumber")); + } catch (UnsupportedEncodingException e) { + Assert.fail(e.getMessage()); + } } } diff --git a/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByIdServletTest.java b/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByIdServletTest.java index f2ff962c5a6..900c41c7df8 100644 --- a/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByIdServletTest.java +++ b/framework/src/test/java/org/tron/core/services/http/GetTransactionInfoByIdServletTest.java @@ -3,20 +3,48 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.tron.common.utils.client.utils.HttpMethed.createRequest; +import com.alibaba.fastjson.JSONObject; +import com.google.protobuf.ByteString; + +import java.io.UnsupportedEncodingException; import javax.annotation.Resource; + import org.apache.http.client.methods.HttpPost; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; import org.tron.common.BaseTest; +import org.tron.common.utils.ByteArray; import org.tron.core.Constant; +import org.tron.core.Wallet; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.TransactionCapsule; +import org.tron.core.capsule.TransactionInfoCapsule; +import org.tron.core.capsule.TransactionRetCapsule; import org.tron.core.config.args.Args; +import org.tron.core.db.TransactionStore; +import org.tron.core.db.TransactionStoreTest; +import org.tron.core.store.TransactionRetStore; +import org.tron.protos.Protocol; +import org.tron.protos.contract.BalanceContract; public class GetTransactionInfoByIdServletTest extends BaseTest { @Resource private GetTransactionInfoByIdServlet getTransactionInfoByIdServlet; + @Resource + private TransactionStore transactionStore; + @Resource + private TransactionRetStore transactionRetStore; + + private static final String OWNER_ADDRESS = + Wallet.getAddressPreFixString() + "abd4b9367799eaa3197fecb144eb71de1e049abc"; + private static final String TO_ADDRESS = + Wallet.getAddressPreFixString() + "abd4b9367799eaa3197fecb144eb71de1e049abc"; + private static final long AMOUNT = 100; + private static final byte[] KEY_1 = TransactionStoreTest.randomBytes(21); static { Args.setParam( @@ -26,11 +54,59 @@ public class GetTransactionInfoByIdServletTest extends BaseTest { ); } + @Before + public void init() { + byte[] blockNum = ByteArray.fromLong(100); + TransactionInfoCapsule transactionInfoCapsule = new TransactionInfoCapsule(); + + transactionInfoCapsule.setId(KEY_1); + transactionInfoCapsule.setFee(1000L); + transactionInfoCapsule.setBlockNumber(100L); + transactionInfoCapsule.setBlockTimeStamp(200L); + + TransactionRetCapsule transactionRetCapsule = new TransactionRetCapsule(); + transactionRetCapsule.addTransactionInfo(transactionInfoCapsule.getInstance()); + chainBaseManager.getTransactionRetStore() + .put(blockNum, transactionRetCapsule); + transactionRetStore.put(blockNum, transactionRetCapsule); + + AccountCapsule owner = new AccountCapsule( + ByteString.copyFrom(ByteArray.fromHexString(OWNER_ADDRESS)), + ByteString.copyFromUtf8("owner"), + Protocol.AccountType.forNumber(1)); + owner.setBalance(1000000L); + + AccountCapsule to = new AccountCapsule( + ByteString.copyFrom(ByteArray.fromHexString(TO_ADDRESS)), + ByteString.copyFromUtf8("to"), + Protocol.AccountType.forNumber(1)); + to.setBalance(1000000L); + + chainBaseManager.getAccountStore().put(owner.createDbKey(), + owner); + chainBaseManager.getAccountStore().put(to.createDbKey(), + to); + BalanceContract.TransferContract transferContract = + getContract(AMOUNT, OWNER_ADDRESS, TO_ADDRESS); + TransactionCapsule transactionCapsule = new TransactionCapsule(transferContract, + chainBaseManager.getAccountStore()); + transactionCapsule.setBlockNum(100L); + transactionStore.put(KEY_1, transactionCapsule); + } + + private BalanceContract.TransferContract getContract(long count, + String owneraddress, String toaddress) { + return BalanceContract.TransferContract.newBuilder() + .setOwnerAddress(ByteString.copyFrom(ByteArray.fromHexString(owneraddress))) + .setToAddress(ByteString.copyFrom(ByteArray.fromHexString(toaddress))) + .setAmount(count) + .build(); + } + @Test public void testGetInfoById() { String jsonParam = "{\"value\" : " - + "\"309b6fa3d01353e46f57dd8a8f276" - + "11f98e392b50d035cef213f2c55225a8bd2\"}"; + + "\"" + ByteArray.toHexString(KEY_1) + "\"}"; MockHttpServletRequest request = createRequest(HttpPost.METHOD_NAME); request.setContentType("application/json"); request.setContent(jsonParam.getBytes(UTF_8)); @@ -38,5 +114,13 @@ public void testGetInfoById() { getTransactionInfoByIdServlet.doPost(request, response); Assert.assertEquals(200, response.getStatus()); + try { + String contentAsString = response.getContentAsString(); + JSONObject jsonObject = JSONObject.parseObject(contentAsString); + Assert.assertEquals(1000, jsonObject.get("fee")); + Assert.assertEquals(100, jsonObject.get("blockNumber")); + } catch (UnsupportedEncodingException e) { + Assert.fail(e.getMessage()); + } } } From ce04da413e3429a02e7cbc3e1d85605b800daa9f Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:06:58 +0800 Subject: [PATCH 39/61] feat(proposal): prevent re-submission after `ALLOW_OLD_REWARD_OPT` is approved (#5683) --- .../org/tron/core/utils/ProposalUtil.java | 4 ++ .../core/actuator/utils/ProposalUtilTest.java | 69 ++++++++++++++++++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java b/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java index cf013266bd0..b38c7135341 100644 --- a/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java +++ b/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java @@ -733,6 +733,10 @@ public static void validator(DynamicPropertiesStore dynamicPropertiesStore, throw new ContractValidateException( "Bad chain parameter id [ALLOW_OLD_REWARD_OPT]"); } + if (dynamicPropertiesStore.allowOldRewardOpt()) { + throw new ContractValidateException( + "[ALLOW_OLD_REWARD_OPT] has been valid, no need to propose again"); + } if (value != 1) { throw new ContractValidateException( "This value[ALLOW_OLD_REWARD_OPT] is only allowed to be 1"); diff --git a/framework/src/test/java/org/tron/core/actuator/utils/ProposalUtilTest.java b/framework/src/test/java/org/tron/core/actuator/utils/ProposalUtilTest.java index 6f4df3cba8f..8866540a028 100644 --- a/framework/src/test/java/org/tron/core/actuator/utils/ProposalUtilTest.java +++ b/framework/src/test/java/org/tron/core/actuator/utils/ProposalUtilTest.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import javax.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.BeforeClass; @@ -12,6 +13,7 @@ import org.tron.common.utils.ByteArray; import org.tron.common.utils.ForkController; import org.tron.core.Constant; +import org.tron.core.capsule.BytesCapsule; import org.tron.core.config.Parameter; import org.tron.core.config.Parameter.ForkBlockVersionEnum; import org.tron.core.config.args.Args; @@ -27,6 +29,11 @@ public class ProposalUtilTest extends BaseTest { private static final String LONG_VALUE_ERROR = "Bad chain parameter value, valid range is [0," + LONG_VALUE + "]"; + @Resource + private DynamicPropertiesStore dynamicPropertiesStore; + + ForkController forkUtils = ForkController.instance(); + /** * Init . */ @@ -60,8 +67,6 @@ public void validProposalTypeCheck() throws ContractValidateException { @Test public void validateCheck() { - DynamicPropertiesStore dynamicPropertiesStore = null; - ForkController forkUtils = ForkController.instance(); long invalidValue = -1; try { @@ -328,6 +333,66 @@ public void validateCheck() { e.getMessage()); } + try { + ProposalUtil.validator(dynamicPropertiesStore, forkUtils, + ProposalType.ALLOW_OLD_REWARD_OPT.getCode(), 2); + Assert.fail(); + } catch (ContractValidateException e) { + Assert.assertEquals( + "Bad chain parameter id [ALLOW_OLD_REWARD_OPT]", + e.getMessage()); + } + hardForkTime = + ((ForkBlockVersionEnum.VERSION_4_7_4.getHardForkTime() - 1) / maintenanceTimeInterval + 1) + * maintenanceTimeInterval; + forkUtils.getManager().getDynamicPropertiesStore() + .saveLatestBlockHeaderTimestamp(hardForkTime + 1); + forkUtils.getManager().getDynamicPropertiesStore() + .statsByVersion(ForkBlockVersionEnum.VERSION_4_7_4.getValue(), stats); + try { + ProposalUtil.validator(dynamicPropertiesStore, forkUtils, + ProposalType.ALLOW_OLD_REWARD_OPT.getCode(), 2); + Assert.fail(); + } catch (ContractValidateException e) { + Assert.assertEquals( + "This value[ALLOW_OLD_REWARD_OPT] is only allowed to be 1", + e.getMessage()); + } + try { + ProposalUtil.validator(dynamicPropertiesStore, forkUtils, + ProposalType.ALLOW_OLD_REWARD_OPT.getCode(), 1); + Assert.fail(); + } catch (ContractValidateException e) { + Assert.assertEquals( + "[ALLOW_NEW_REWARD] proposal must be approved " + + "before [ALLOW_OLD_REWARD_OPT] can be proposed", + e.getMessage()); + } + dynamicPropertiesStore.saveCurrentCycleNumber(0); + dynamicPropertiesStore.saveNewRewardAlgorithmEffectiveCycle(); + dynamicPropertiesStore.saveAllowNewReward(1); + try { + ProposalUtil.validator(dynamicPropertiesStore, forkUtils, + ProposalType.ALLOW_OLD_REWARD_OPT.getCode(), 1); + Assert.fail(); + } catch (ContractValidateException e) { + Assert.assertEquals( + "no need old reward opt, ALLOW_NEW_REWARD from start cycle 1", + e.getMessage()); + } + dynamicPropertiesStore.put("NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE".getBytes(), + new BytesCapsule(ByteArray.fromLong(4000))); + dynamicPropertiesStore.saveAllowOldRewardOpt(1); + try { + ProposalUtil.validator(dynamicPropertiesStore, forkUtils, + ProposalType.ALLOW_OLD_REWARD_OPT.getCode(), 1); + Assert.fail(); + } catch (ContractValidateException e) { + Assert.assertEquals( + "[ALLOW_OLD_REWARD_OPT] has been valid, no need to propose again", + e.getMessage()); + } + forkUtils.getManager().getDynamicPropertiesStore() .statsByVersion(ForkBlockVersionEnum.ENERGY_LIMIT.getValue(), stats); forkUtils.reset(); From f02d6bfcd2adb64cf94c9d3b6215bdc7864c60f4 Mon Sep 17 00:00:00 2001 From: Kayle Date: Thu, 18 Jan 2024 21:05:51 +0800 Subject: [PATCH 40/61] test(case): remove useless code (#5679) --- .../java/org/tron/core/db/ManagerTest.java | 48 ++++++++++++++++--- .../tron/core/db/TransactionStoreTest.java | 2 +- .../tron/core/zksnark/NoteEncDecryTest.java | 2 - .../core/zksnark/ShieldedReceiveTest.java | 5 +- .../src/test/resources/config-localtest.conf | 10 ---- 5 files changed, 45 insertions(+), 22 deletions(-) diff --git a/framework/src/test/java/org/tron/core/db/ManagerTest.java b/framework/src/test/java/org/tron/core/db/ManagerTest.java index bd79f7d9776..053647cc25a 100755 --- a/framework/src/test/java/org/tron/core/db/ManagerTest.java +++ b/framework/src/test/java/org/tron/core/db/ManagerTest.java @@ -580,14 +580,24 @@ public void pushSwitchFork() TaposException, ReceiptCheckErrException, TooBigTransactionException, AccountResourceInsufficientException, EventBloomException { - String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62"; - String key2 = "c85ef7d79691fe79573b1a7064c19c1a9819ebdbd1faaab1a8ec92344438aaf4"; + String key = PublicMethod.getRandomPrivateKey(); + String key2 = PublicMethod.getRandomPrivateKey(); byte[] privateKey = ByteArray.fromHexString(key); final ECKey ecKey = ECKey.fromPrivate(privateKey); byte[] address = ecKey.getAddress(); + + ByteString addressByte = ByteString.copyFrom(address); + AccountCapsule accountCapsule = + new AccountCapsule(Protocol.Account.newBuilder() + .setAddress(addressByte).build()); + chainManager.getAccountStore() + .put(addressByte.toByteArray(), accountCapsule); + WitnessCapsule sr1 = new WitnessCapsule( ByteString.copyFrom(address), "www.tron.net/first"); sr1.setVoteCount(1000000000L); + + byte[] privateKey2 = ByteArray.fromHexString(key2); final ECKey ecKey2 = ECKey.fromPrivate(privateKey2); byte[] address2 = ecKey2.getAddress(); @@ -703,13 +713,23 @@ public void fork() Args.setParam(new String[]{"--witness"}, Constant.TEST_CONF); long size = chainManager.getBlockStore().size(); // System.out.print("block store size:" + size + "\n"); - String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62"; + String key = PublicMethod.getRandomPrivateKey(); byte[] privateKey = ByteArray.fromHexString(key); final ECKey ecKey = ECKey.fromPrivate(privateKey); byte[] address = ecKey.getAddress(); + + ByteString addressByte = ByteString.copyFrom(address); + AccountCapsule accountCapsule = + new AccountCapsule(Protocol.Account.newBuilder() + .setAddress(addressByte).build()); + chainManager.getAccountStore() + .put(addressByte.toByteArray(), accountCapsule); + + WitnessCapsule witnessCapsule = new WitnessCapsule(ByteString.copyFrom(address)); chainManager.getWitnessScheduleStore().saveActiveWitnesses(new ArrayList<>()); chainManager.addWitness(ByteString.copyFrom(address)); + chainManager.getWitnessStore().put(address, witnessCapsule); Block block = getSignedBlock(witnessCapsule.getAddress(), 1533529947843L, privateKey); @@ -822,14 +842,20 @@ public void doNotSwitch() Args.setParam(new String[]{"--witness"}, Constant.TEST_CONF); long size = chainManager.getBlockStore().size(); System.out.print("block store size:" + size + "\n"); - String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62"; + String key = PublicMethod.getRandomPrivateKey(); byte[] privateKey = ByteArray.fromHexString(key); final ECKey ecKey = ECKey.fromPrivate(privateKey); byte[] address = ecKey.getAddress(); - + ByteString addressByte = ByteString.copyFrom(address); + AccountCapsule accountCapsule = + new AccountCapsule(Protocol.Account.newBuilder() + .setAddress(addressByte).build()); + chainManager.getAccountStore() + .put(addressByte.toByteArray(), accountCapsule); WitnessCapsule witnessCapsule = new WitnessCapsule(ByteString.copyFrom(address)); chainManager.getWitnessScheduleStore().saveActiveWitnesses(new ArrayList<>()); chainManager.addWitness(ByteString.copyFrom(address)); + chainManager.getWitnessStore().put(address, witnessCapsule); Block block = getSignedBlock(witnessCapsule.getAddress(), 1533529947843L, privateKey); dbManager.pushBlock(new BlockCapsule(block)); @@ -928,14 +954,22 @@ public void switchBack() Args.setParam(new String[]{"--witness"}, Constant.TEST_CONF); long size = chainManager.getBlockStore().size(); System.out.print("block store size:" + size + "\n"); - String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62"; + String key = PublicMethod.getRandomPrivateKey();; byte[] privateKey = ByteArray.fromHexString(key); final ECKey ecKey = ECKey.fromPrivate(privateKey); byte[] address = ecKey.getAddress(); + + ByteString addressByte = ByteString.copyFrom(address); + AccountCapsule accountCapsule = + new AccountCapsule(Protocol.Account.newBuilder() + .setAddress(addressByte).build()); + chainManager.getAccountStore() + .put(addressByte.toByteArray(), accountCapsule); + WitnessCapsule witnessCapsule = new WitnessCapsule(ByteString.copyFrom(address)); chainManager.getWitnessScheduleStore().saveActiveWitnesses(new ArrayList<>()); chainManager.addWitness(ByteString.copyFrom(address)); - + chainManager.getWitnessStore().put(address, witnessCapsule); Block block = getSignedBlock(witnessCapsule.getAddress(), 1533529947843L, privateKey); dbManager.pushBlock(new BlockCapsule(block)); diff --git a/framework/src/test/java/org/tron/core/db/TransactionStoreTest.java b/framework/src/test/java/org/tron/core/db/TransactionStoreTest.java index 1edc4aca756..deeb135373f 100644 --- a/framework/src/test/java/org/tron/core/db/TransactionStoreTest.java +++ b/framework/src/test/java/org/tron/core/db/TransactionStoreTest.java @@ -112,7 +112,7 @@ private VoteWitnessContract getVoteWitnessContract(String address, String votead public void getTransactionTest() throws BadItemException, ItemNotFoundException { final BlockStore blockStore = chainBaseManager.getBlockStore(); final TransactionStore trxStore = chainBaseManager.getTransactionStore(); - String key = "f31db24bfbd1a2ef19beddca0a0fa37632eded9ac666a05d3bd925f01dde1f62"; + String key = PublicMethod.getRandomPrivateKey();; BlockCapsule blockCapsule = new BlockCapsule( diff --git a/framework/src/test/java/org/tron/core/zksnark/NoteEncDecryTest.java b/framework/src/test/java/org/tron/core/zksnark/NoteEncDecryTest.java index e59060540e9..3c3fb14b2b1 100644 --- a/framework/src/test/java/org/tron/core/zksnark/NoteEncDecryTest.java +++ b/framework/src/test/java/org/tron/core/zksnark/NoteEncDecryTest.java @@ -23,7 +23,6 @@ public class NoteEncDecryTest extends BaseTest { private static final String FROM_ADDRESS; - private static final String ADDRESS_ONE_PRIVATE_KEY; private static final long OWNER_BALANCE = 100_000_000; private static final long FROM_AMOUNT = 110_000_000; private static final long tokenId = 1; @@ -42,7 +41,6 @@ public class NoteEncDecryTest extends BaseTest { static { Args.setParam(new String[]{"--output-directory", dbPath()}, "config-localtest.conf"); FROM_ADDRESS = Wallet.getAddressPreFixString() + "a7d8a35b260395c14aa456297662092ba3b76fc0"; - ADDRESS_ONE_PRIVATE_KEY = "7f7f701e94d4f1dd60ee5205e7ea8ee31121427210417b608a6b2e96433549a7"; } /** diff --git a/framework/src/test/java/org/tron/core/zksnark/ShieldedReceiveTest.java b/framework/src/test/java/org/tron/core/zksnark/ShieldedReceiveTest.java index c963a92bb9d..f3ad1f36cd1 100755 --- a/framework/src/test/java/org/tron/core/zksnark/ShieldedReceiveTest.java +++ b/framework/src/test/java/org/tron/core/zksnark/ShieldedReceiveTest.java @@ -25,6 +25,7 @@ import org.tron.common.crypto.ECKey; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; +import org.tron.common.utils.PublicMethod; import org.tron.common.utils.Sha256Hash; import org.tron.common.utils.client.utils.TransactionUtils; import org.tron.common.zksnark.IncrementalMerkleTreeContainer; @@ -124,8 +125,8 @@ public class ShieldedReceiveTest extends BaseTest { static { Args.setParam(new String[]{"--output-directory", dbPath()}, "config-localtest.conf"); - FROM_ADDRESS = Wallet.getAddressPreFixString() + "a7d8a35b260395c14aa456297662092ba3b76fc0"; - ADDRESS_ONE_PRIVATE_KEY = "7f7f701e94d4f1dd60ee5205e7ea8ee31121427210417b608a6b2e96433549a7"; + ADDRESS_ONE_PRIVATE_KEY = PublicMethod.getRandomPrivateKey(); + FROM_ADDRESS = PublicMethod.getHexAddressByPrivateKey(ADDRESS_ONE_PRIVATE_KEY);; } /** diff --git a/framework/src/test/resources/config-localtest.conf b/framework/src/test/resources/config-localtest.conf index ff0fe8bf9d9..fc35ca9b66a 100644 --- a/framework/src/test/resources/config-localtest.conf +++ b/framework/src/test/resources/config-localtest.conf @@ -188,8 +188,6 @@ genesis.block = { accountType = "AssetIssue" address = "TJCnKsPa7y5okkXvQAidZBzqx3QyQ6sxMW" balance = "25000000000000000" - #priKey = D95611A9AF2A2A45359106222ED1AFED48853D9A44DEFF8DC7913F5CBA727366 - #password = 2VYRqa8qKkU1kQYiLtGv7UiFPZpE3v+Nx5E/XLpyc2Y= }, # the account of payment @@ -198,8 +196,6 @@ genesis.block = { accountType = "AssetIssue" address = "TGehVcNhud84JDCGrNHKVz9jEAVKUpbuiv" balance = "10000000000000000" - #priKey = cba92a516ea09f620a16ff7ee95ce0df1d56550a8babe9964981a7144c8a784a - #password = y6kqUW6gn2IKFv9+6Vzg3x1WVQqLq+mWSYGnFEyKeEo= }, # the account of coin burn @@ -208,8 +204,6 @@ genesis.block = { accountType = "AssetIssue" address = "THKrowiEfCe8evdbaBzDDvQjM5DGeB3s3F" balance = "-9223372036854775808" - #priKey = 8E812436A0E3323166E1F0E8BA79E19E217B2C4A53C970D4CCA0CFB1078979DF - #password = joEkNqDjMjFm4fDounnhniF7LEpTyXDUzKDPsQeJed8= } ] @@ -218,27 +212,23 @@ genesis.block = { address: TN3zfjYUmMFK3ZsHSsrdJoNRtGkQmZLBLz url = "http://Test.org", voteCount = 106 - #priKey = f4df789d3210ac881cb900464dd30409453044d2777060a0c391cbdf4c6a4f57 6666 }, // { // address: TPrLL5ckUdMaPNgJYmGv23qtYjBE34aBf8 // url = "http://Mercury.org", // voteCount = 105 - // #priKey = f5583fd20e13073900a513f333ed13db8c9e83e7e3cf37e74adacef96c5afeaa 7777 // }, // { // address: TEZBh76rouEQpB2zqYVopbRXGx7RfyWorT // #address: 27TfVERREG3FeWMHEAQ95tWHG4sb3ANn3Qe // url = "http://Venus.org", // voteCount = 104 - // #priKey = 9f5c5e48bf87cf92017313082e8cf0f58ccfce423097f0fcebf801695fc99bd4 8888 // }, // { // address: TN27wbfCLEN1gP2PZAxHgU3QZrntsLyxdj // #address: 27b8RUuyZnNPFNZGct2bZkNu9MnGWNAdH3Z // url = "http://Earth.org", // voteCount = 103 - // #priKey = 6781f44d9a2083b14fad1702b8e9ba82749162b795e2fc3f136192fc63f80de2 9999 // }, ] From 24da38479eb35577f9285121c1de1e23af941d79 Mon Sep 17 00:00:00 2001 From: liukai Date: Mon, 22 Jan 2024 11:33:01 +0800 Subject: [PATCH 41/61] feat(version): update version --- framework/src/main/java/org/tron/program/Version.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/program/Version.java b/framework/src/main/java/org/tron/program/Version.java index a01eb714297..6f26752524c 100644 --- a/framework/src/main/java/org/tron/program/Version.java +++ b/framework/src/main/java/org/tron/program/Version.java @@ -4,7 +4,7 @@ public class Version { public static final String VERSION_NAME = "GreatVoyage-v4.7.3-5-g788136ebe"; public static final String VERSION_CODE = "18180"; - private static final String VERSION = "4.7.3.1"; + private static final String VERSION = "4.7.4"; public static String getVersion() { return VERSION; From b5c6cbd3f756450f59da51467c52aef5058c2d47 Mon Sep 17 00:00:00 2001 From: lurais <107600021+lurais@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:55:45 +0800 Subject: [PATCH 42/61] feat(log):optimize fork block version log (#5687) Co-authored-by: morgan.peng --- .../src/main/java/org/tron/common/utils/ForkController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chainbase/src/main/java/org/tron/common/utils/ForkController.java b/chainbase/src/main/java/org/tron/common/utils/ForkController.java index c3db883a011..7cbac28e781 100644 --- a/chainbase/src/main/java/org/tron/common/utils/ForkController.java +++ b/chainbase/src/main/java/org/tron/common/utils/ForkController.java @@ -78,7 +78,7 @@ private boolean passOld(int version) { private boolean passNew(int version) { ForkBlockVersionEnum versionEnum = ForkBlockVersionEnum.getForkBlockVersionEnum(version); if (versionEnum == null) { - logger.error("Not exist block version: {}.", version); + logger.warn("Not exist block version: {}.", version); return false; } long latestBlockTime = manager.getDynamicPropertiesStore().getLatestBlockHeaderTimestamp(); From 245ddfb0167aad70fae35eb13e16843ec2ebf392 Mon Sep 17 00:00:00 2001 From: running-tomato <31307926+tomatoishealthy@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:22:07 +0800 Subject: [PATCH 43/61] fix(dependency): mock jar version upgrade (#5689) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e3611a27a73..a56be97afa1 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,7 @@ subprojects { testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' testImplementation group: 'junit', name: 'junit', version: '4.13.2' - testImplementation "org.mockito:mockito-core:2.1.0" + testImplementation "org.mockito:mockito-core:2.13.0" } task sourcesJar(type: Jar, dependsOn: classes) { From f0a46ff1fbb0160d93fdd1bc512af3f01bed9995 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Tue, 23 Jan 2024 10:04:34 +0800 Subject: [PATCH 44/61] fix(api): fix the bug of determine if triggerConstantContract succeeds (#5688) * fix(api):fix the bug of determine if get scale factor succeed * fix(api):fix bug of isShieldedTRC20NoteSpent --- .../src/main/java/org/tron/core/Wallet.java | 14 +++--- .../tron/core/capsule/utils/RLPListTest.java | 43 +++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index cd2e26aa6f7..6e5c17417b7 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -3862,11 +3862,12 @@ private boolean isShieldedTRC20NoteSpent(GrpcAPI.Note note, long pos, byte[] ak, TransactionExtention.Builder trxExtBuilder = TransactionExtention.newBuilder(); Return.Builder retBuilder = Return.newBuilder(); TransactionExtention trxExt; + Transaction trx; try { TransactionCapsule trxCap = createTransactionCapsule(trigger, ContractType.TriggerSmartContract); - Transaction trx = triggerConstantContract(trigger, trxCap, trxExtBuilder, retBuilder); + trx = triggerConstantContract(trigger, trxCap, trxExtBuilder, retBuilder); retBuilder.setResult(true).setCode(response_code.SUCCESS); trxExtBuilder.setTransaction(trx); @@ -3889,10 +3890,10 @@ private boolean isShieldedTRC20NoteSpent(GrpcAPI.Note note, long pos, byte[] ak, logger.warn("unknown exception caught: " + e.getMessage(), e); } finally { trxExt = trxExtBuilder.build(); + trx = trxExt.getTransaction(); } - String code = trxExt.getResult().getCode().toString(); - if ("SUCCESS".equals(code)) { + if (code.SUCESS == trx.getRet(0).getRet()) { List list = trxExt.getConstantResultList(); byte[] listBytes = new byte[0]; for (ByteString bs : list) { @@ -4126,11 +4127,12 @@ public byte[] getShieldedContractScalingFactor(byte[] contractAddress) TransactionExtention.Builder trxExtBuilder = TransactionExtention.newBuilder(); Return.Builder retBuilder = Return.newBuilder(); TransactionExtention trxExt; + Transaction trx; try { TransactionCapsule trxCap = createTransactionCapsule(trigger, ContractType.TriggerSmartContract); - Transaction trx = triggerConstantContract(trigger, trxCap, trxExtBuilder, retBuilder); + trx = triggerConstantContract(trigger, trxCap, trxExtBuilder, retBuilder); retBuilder.setResult(true).setCode(response_code.SUCCESS); trxExtBuilder.setTransaction(trx); @@ -4153,10 +4155,10 @@ public byte[] getShieldedContractScalingFactor(byte[] contractAddress) logger.warn("Unknown exception caught: " + e.getMessage(), e); } finally { trxExt = trxExtBuilder.build(); + trx = trxExt.getTransaction(); } - String code = trxExt.getResult().getCode().toString(); - if ("SUCCESS".equals(code)) { + if (code.SUCESS == trx.getRet(0).getRet()) { List list = trxExt.getConstantResultList(); byte[] listBytes = new byte[0]; for (ByteString bs : list) { diff --git a/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java b/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java index 160ecd8f6dd..cba4a7d8040 100644 --- a/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java +++ b/framework/src/test/java/org/tron/core/capsule/utils/RLPListTest.java @@ -2,8 +2,12 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.math.BigInteger; +import java.util.Random; +import org.bouncycastle.util.BigIntegers; import org.junit.Assert; import org.junit.Test; +import org.tron.common.utils.Value; public class RLPListTest { @@ -34,5 +38,44 @@ public void testToBytes() byte[] aBytes = new byte[10]; byte[] bBytes = (byte[]) method.invoke(RLP.class, aBytes); Assert.assertArrayEquals(aBytes, bBytes); + + int i = new Random().nextInt(); + byte[] cBytes = BigIntegers.asUnsignedByteArray(BigInteger.valueOf(i)); + byte[] dBytes = (byte[]) method.invoke(RLP.class, i); + Assert.assertArrayEquals(cBytes, dBytes); + + long j = new Random().nextInt(); + byte[] eBytes = BigIntegers.asUnsignedByteArray(BigInteger.valueOf(j)); + byte[] fBytes = (byte[]) method.invoke(RLP.class, j); + Assert.assertArrayEquals(eBytes, fBytes); + + String test = "testA"; + byte[] gBytes = test.getBytes(); + byte[] hBytes = (byte[]) method.invoke(RLP.class, test); + Assert.assertArrayEquals(gBytes, hBytes); + + BigInteger bigInteger = BigInteger.valueOf(100); + byte[] iBytes = BigIntegers.asUnsignedByteArray(bigInteger); + byte[] jBytes = (byte[]) method.invoke(RLP.class, bigInteger); + Assert.assertArrayEquals(iBytes, jBytes); + + Value v = new Value(new byte[0]); + byte[] kBytes = v.asBytes(); + byte[] lBytes = (byte[]) method.invoke(RLP.class, v); + Assert.assertArrayEquals(kBytes, lBytes); + + char c = 'a'; + try { + method.invoke(RLP.class, c); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(true); + } + } + + @Test + public void testEncode() { + byte[] aBytes = RLP.encode(new byte[1]); + Assert.assertEquals(1, aBytes.length); } } From 8960a6bf6c49ad535b8a922cfac0ef1ca4cd5a08 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Tue, 23 Jan 2024 16:56:48 +0800 Subject: [PATCH 45/61] feat(version):update libp2p version to 2.2.0 (#5692) --- common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/build.gradle b/common/build.gradle index ba1c431e74c..2e5413269d9 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -46,7 +46,7 @@ dependencies { compile 'org.aspectj:aspectjrt:1.8.13' compile 'org.aspectj:aspectjweaver:1.8.13' compile 'org.aspectj:aspectjtools:1.8.13' - compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v2.2.1',{ + compile group: 'io.github.tronprotocol', name: 'libp2p', version: '2.2.0',{ exclude group: 'io.grpc', module: 'grpc-context' exclude group: 'io.grpc', module: 'grpc-core' exclude group: 'io.grpc', module: 'grpc-netty' From 931de4438f73f73a62baf650a369432282c9a2e9 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Wed, 24 Jan 2024 14:26:09 +0800 Subject: [PATCH 46/61] feat(dependency):update libp2p version to test-v2.2.2 --- common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/build.gradle b/common/build.gradle index 2e5413269d9..8fbd2954f47 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -46,7 +46,7 @@ dependencies { compile 'org.aspectj:aspectjrt:1.8.13' compile 'org.aspectj:aspectjweaver:1.8.13' compile 'org.aspectj:aspectjtools:1.8.13' - compile group: 'io.github.tronprotocol', name: 'libp2p', version: '2.2.0',{ + compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v2.2.2',{ exclude group: 'io.grpc', module: 'grpc-context' exclude group: 'io.grpc', module: 'grpc-core' exclude group: 'io.grpc', module: 'grpc-netty' From 9911b054a5a22960d32e13f68144ff7a71b72a1e Mon Sep 17 00:00:00 2001 From: wubin01 Date: Thu, 25 Jan 2024 15:52:14 +0800 Subject: [PATCH 47/61] feat(dependency): update libp2p version to 2.2.1 --- common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/build.gradle b/common/build.gradle index 8fbd2954f47..6c1545e5d13 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -46,7 +46,7 @@ dependencies { compile 'org.aspectj:aspectjrt:1.8.13' compile 'org.aspectj:aspectjweaver:1.8.13' compile 'org.aspectj:aspectjtools:1.8.13' - compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v2.2.2',{ + compile group: 'io.github.tronprotocol', name: 'libp2p', version: '2.2.1',{ exclude group: 'io.grpc', module: 'grpc-context' exclude group: 'io.grpc', module: 'grpc-core' exclude group: 'io.grpc', module: 'grpc-netty' From baddd4f90387be76253fffb39b6be294b902de0e Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Thu, 25 Jan 2024 17:55:19 +0800 Subject: [PATCH 48/61] feat(log): remove unused log for getnodeinfo api (#5697) --- framework/src/main/java/org/tron/core/db/Manager.java | 4 ++++ .../src/main/java/org/tron/core/services/NodeInfoService.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 445c48fdd94..63bbef9ff7f 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -1914,6 +1914,10 @@ public long getSyncBeginNumber() { chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() - revokingStore.size(), chainBaseManager.getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); + return this.fetchSyncBeginNumber(); + } + + public long fetchSyncBeginNumber() { return chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() - revokingStore.size(); } diff --git a/framework/src/main/java/org/tron/core/services/NodeInfoService.java b/framework/src/main/java/org/tron/core/services/NodeInfoService.java index ddf72c64410..b55ce96748e 100644 --- a/framework/src/main/java/org/tron/core/services/NodeInfoService.java +++ b/framework/src/main/java/org/tron/core/services/NodeInfoService.java @@ -195,7 +195,7 @@ private void setConfigNodeInfo(NodeInfo nodeInfo) { } protected void setBlockInfo(NodeInfo nodeInfo) { - nodeInfo.setBeginSyncNum(dbManager.getSyncBeginNumber()); + nodeInfo.setBeginSyncNum(dbManager.fetchSyncBeginNumber()); nodeInfo.setBlock(chainBaseManager.getHeadBlockId().getString()); nodeInfo.setSolidityBlock(chainBaseManager.getSolidBlockId().getString()); } From cbf24173f47cc3f58df6b4223370f4601cb0a574 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Fri, 26 Jan 2024 10:57:19 +0800 Subject: [PATCH 49/61] fix the bug of concurrent usage of toString in TransactionCapsule (#5696) --- .../tron/core/capsule/TransactionCapsule.java | 3 +-- .../actuator/utils/TransactionUtilTest.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java b/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java index 9598fd99a6b..5d5a5d5b462 100755 --- a/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java +++ b/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java @@ -102,7 +102,6 @@ public class TransactionCapsule implements ProtoCapsule { @Setter private TransactionTrace trxTrace; - private StringBuilder toStringBuff = new StringBuilder(); @Getter @Setter private long time; @@ -738,7 +737,7 @@ public Transaction getInstance() { @Override public String toString() { - + StringBuilder toStringBuff = new StringBuilder(); toStringBuff.setLength(0); toStringBuff.append("TransactionCapsule \n[ "); diff --git a/framework/src/test/java/org/tron/core/actuator/utils/TransactionUtilTest.java b/framework/src/test/java/org/tron/core/actuator/utils/TransactionUtilTest.java index 9d56876a4da..3346a1aead5 100644 --- a/framework/src/test/java/org/tron/core/actuator/utils/TransactionUtilTest.java +++ b/framework/src/test/java/org/tron/core/actuator/utils/TransactionUtilTest.java @@ -14,6 +14,7 @@ import com.google.protobuf.ByteString; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; @@ -431,4 +432,22 @@ public void estimateConsumeBandWidthSizeCorner() { long actual = TransactionUtil.estimateConsumeBandWidthSize(dps, balance); Assert.assertEquals(expected, actual); } + + @Test + public void testConcurrentToString() throws InterruptedException { + Transaction.Builder builder = Transaction.newBuilder(); + TransactionCapsule trx = new TransactionCapsule(builder.build()); + List threadList = new ArrayList<>(); + int n = 10; + for (int i = 0; i < n; i++) { + threadList.add(new Thread(() -> trx.toString())); + } + for (int i = 0; i < n; i++) { + threadList.get(i).start(); + } + for (int i = 0; i < n; i++) { + threadList.get(i).join(); + } + Assert.assertTrue(true); + } } From 976d6f7a328db07e8fb59d7dd4bf77d9c12561a1 Mon Sep 17 00:00:00 2001 From: allen <56535423+jwrct@users.noreply.github.com> Date: Fri, 26 Jan 2024 21:36:03 +0800 Subject: [PATCH 50/61] fix(pbft): fix concurrent access issue of srPbftMessage (#5703) * fix(pbft): fix the concurrent access issue of srPbftMessage --- .../main/java/org/tron/consensus/pbft/PbftMessageHandle.java | 4 ++-- framework/src/test/java/org/tron/core/pbft/PbftTest.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/consensus/src/main/java/org/tron/consensus/pbft/PbftMessageHandle.java b/consensus/src/main/java/org/tron/consensus/pbft/PbftMessageHandle.java index ea7b00802b2..523ffac4d61 100644 --- a/consensus/src/main/java/org/tron/consensus/pbft/PbftMessageHandle.java +++ b/consensus/src/main/java/org/tron/consensus/pbft/PbftMessageHandle.java @@ -262,7 +262,7 @@ public boolean isSyncing() { } //Cleanup related status - private void remove(String no) { + private synchronized void remove(String no) { String pre = String.valueOf(no) + "_"; preVotes.remove(no); pareVoteMap.keySet().removeIf(vp -> StringUtils.startsWith(vp, pre)); @@ -285,7 +285,7 @@ private void remove(String no) { //just try once if (srPbftMessage != null && StringUtils.equals(no, srPbftMessage.getNo())) { try { - Thread.sleep(100); + wait(100); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (Exception e) { diff --git a/framework/src/test/java/org/tron/core/pbft/PbftTest.java b/framework/src/test/java/org/tron/core/pbft/PbftTest.java index a5d74ad7a37..33a46516988 100644 --- a/framework/src/test/java/org/tron/core/pbft/PbftTest.java +++ b/framework/src/test/java/org/tron/core/pbft/PbftTest.java @@ -29,6 +29,7 @@ public void testPbftSrMessage() { ByteString.copyFrom(ByteArray.fromHexString("41df309fef25b311e7895562bd9e11aab2a58816d2"))); PbftMessage pbftSrMessage = PbftMessage .prePrepareSRLMsg(blockCapsule, srList, 1, miner); + PbftMessage.fullNodePrePrepareSRLMsg(blockCapsule, srList, 1); System.out.println(pbftSrMessage); } From 5acf3efb97afe8b2f82d2bc6643c47843e133a40 Mon Sep 17 00:00:00 2001 From: chengtx01 Date: Sat, 27 Jan 2024 21:02:14 +0800 Subject: [PATCH 51/61] feat(dependency): update libp2p version to test-v2.2.3 --- common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/build.gradle b/common/build.gradle index 6c1545e5d13..489a2fa9d49 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -46,7 +46,7 @@ dependencies { compile 'org.aspectj:aspectjrt:1.8.13' compile 'org.aspectj:aspectjweaver:1.8.13' compile 'org.aspectj:aspectjtools:1.8.13' - compile group: 'io.github.tronprotocol', name: 'libp2p', version: '2.2.1',{ + compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v2.2.3',{ exclude group: 'io.grpc', module: 'grpc-context' exclude group: 'io.grpc', module: 'grpc-core' exclude group: 'io.grpc', module: 'grpc-netty' From a578e85fbfede47bf4a7fe011f2646fabf280e3c Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:40:32 +0800 Subject: [PATCH 52/61] feat(api): adjust API services to start first at startup (#5711) --- .../java/org/tron/common/application/ApplicationImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java index 9133fddf434..3cb75cb1e24 100644 --- a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java +++ b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java @@ -54,13 +54,13 @@ public void initServices(CommonParameter parameter) { * start up the app. */ public void startup() { + this.initServices(Args.getInstance()); + this.startServices(); if ((!Args.getInstance().isSolidityNode()) && (!Args.getInstance().isP2pDisable())) { tronNetService.start(); } consensusService.start(); MetricsUtil.init(); - this.initServices(Args.getInstance()); - this.startServices(); } @Override From dcf1103d5ea195c80a0ded3b4f49856e9368b61d Mon Sep 17 00:00:00 2001 From: allen <56535423+jwrct@users.noreply.github.com> Date: Mon, 29 Jan 2024 14:04:32 +0800 Subject: [PATCH 53/61] Revert "feat(dependency): update libp2p version to test-v2.2.3" (#5712) --- common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/build.gradle b/common/build.gradle index 489a2fa9d49..6c1545e5d13 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -46,7 +46,7 @@ dependencies { compile 'org.aspectj:aspectjrt:1.8.13' compile 'org.aspectj:aspectjweaver:1.8.13' compile 'org.aspectj:aspectjtools:1.8.13' - compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v2.2.3',{ + compile group: 'io.github.tronprotocol', name: 'libp2p', version: '2.2.1',{ exclude group: 'io.grpc', module: 'grpc-context' exclude group: 'io.grpc', module: 'grpc-core' exclude group: 'io.grpc', module: 'grpc-netty' From 96afa1587e62afad2e023928ec3aae60b7d718dc Mon Sep 17 00:00:00 2001 From: running-tomato <31307926+tomatoishealthy@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:58:17 +0800 Subject: [PATCH 54/61] fix(conf): fix the config location of the RPC reflection servive (#5739) --- framework/src/main/resources/config.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index 019b6302911..f0479a29cb2 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -243,6 +243,9 @@ node { # Transactions can only be broadcast if the number of effective connections is reached. minEffectiveConnection = 1 + + # The switch of the reflection service, effective for all gRPC services + # reflectionService = true } # number of solidity thread in the FullNode. @@ -285,8 +288,6 @@ node { # "getnowblock2" # ] - # The switch of the reflection service, effective for all gRPC services - # reflectionService = true } ## rate limiter config From fa46c4f023562e75b228d43b575f4feaf89b3c08 Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:58:59 +0800 Subject: [PATCH 55/61] feat(*): optimize old rewards withdrawal (#5742) * remove useNewRewardAlgorithmFromStart * code clear * exit when the calculation task gets an exception * tools: add `db root` to calculate the data root * make reward-vi root configurable * fix race-condition for Merkel root calculation * optimize proposal design --- .../org/tron/core/utils/ProposalUtil.java | 6 +- .../tron/core/capsule/utils/MerkleTree.java | 2 + .../core/db/common/iterator/DBIterator.java | 6 - .../db/common/iterator/RockStoreIterator.java | 2 - .../db/common/iterator/StoreIterator.java | 2 - .../tron/core/service/RewardViCalService.java | 59 +++-- .../core/store/DynamicPropertiesStore.java | 17 +- .../tron/common/prometheus/MetricKeys.java | 2 - .../common/prometheus/MetricsHistogram.java | 2 - .../org/tron/common/utils/MerkleRoot.java | 68 ++++++ .../org/tron/core/config/args/Storage.java | 19 ++ .../java/org/tron/core/config/args/Args.java | 15 +- framework/src/main/resources/config.conf | 7 + .../core/actuator/utils/ProposalUtilTest.java | 14 +- .../core/capsule/utils/MerkleTreeTest.java | 35 ++- .../org/tron/core/config/args/ArgsTest.java | 11 + .../tron/core/services/ComputeRewardTest.java | 34 +++ .../core/services/ProposalServiceTest.java | 9 - framework/src/test/resources/args-test.conf | 224 ++++++++++++++++++ .../resources/config-test-storagetest.conf | 2 + framework/src/test/resources/config-test.conf | 6 + .../src/main/java/org/tron/plugins/Db.java | 3 +- .../main/java/org/tron/plugins/DbRoot.java | 99 ++++++++ .../org/tron/plugins/utils/MerkleRoot.java | 68 ++++++ .../org/tron/plugins/utils/Sha256Hash.java | 10 +- .../tron/plugins/utils/db/DBInterface.java | 2 + .../org/tron/plugins/utils/db/DbTool.java | 18 +- .../tron/plugins/utils/db/LevelDBImpl.java | 7 +- .../tron/plugins/utils/db/RocksDBImpl.java | 7 +- .../java/org/tron/plugins/DbRootTest.java | 82 +++++++ 30 files changed, 738 insertions(+), 100 deletions(-) create mode 100644 common/src/main/java/org/tron/common/utils/MerkleRoot.java create mode 100644 framework/src/test/resources/args-test.conf create mode 100644 plugins/src/main/java/org/tron/plugins/DbRoot.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/MerkleRoot.java create mode 100644 plugins/src/test/java/org/tron/plugins/DbRootTest.java diff --git a/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java b/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java index b38c7135341..cfec5b09d96 100644 --- a/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java +++ b/actuator/src/main/java/org/tron/core/utils/ProposalUtil.java @@ -743,13 +743,9 @@ public static void validator(DynamicPropertiesStore dynamicPropertiesStore, } if (!dynamicPropertiesStore.useNewRewardAlgorithm()) { throw new ContractValidateException( - "[ALLOW_NEW_REWARD] proposal must be approved " + "[ALLOW_NEW_REWARD] or [ALLOW_TVM_VOTE] proposal must be approved " + "before [ALLOW_OLD_REWARD_OPT] can be proposed"); } - if (dynamicPropertiesStore.useNewRewardAlgorithmFromStart()) { - throw new ContractValidateException( - "no need old reward opt, ALLOW_NEW_REWARD from start cycle 1"); - } break; } default: diff --git a/chainbase/src/main/java/org/tron/core/capsule/utils/MerkleTree.java b/chainbase/src/main/java/org/tron/core/capsule/utils/MerkleTree.java index 47ac45c9fb8..94d22f4b474 100644 --- a/chainbase/src/main/java/org/tron/core/capsule/utils/MerkleTree.java +++ b/chainbase/src/main/java/org/tron/core/capsule/utils/MerkleTree.java @@ -5,10 +5,12 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import lombok.Getter; +import net.jcip.annotations.NotThreadSafe; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.Sha256Hash; @Getter +@NotThreadSafe public class MerkleTree { private static volatile MerkleTree instance; diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java index ec8d7fd85be..afbacac35db 100755 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java @@ -16,12 +16,6 @@ public interface DBIterator extends Iterator>, AutoCloseab void seekToLast(); - default UnmodifiableIterator> prefixQueryAfterThat - (byte[] key, byte[] afterThat) { - this.seek(afterThat == null ? key : afterThat); - return Iterators.filter(this, entry -> Bytes.indexOf(entry.getKey(), key) == 0); - } - /** * An iterator is either positioned at a key/value pair, or * not valid. This method returns true iff the iterator is valid. diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java index 1438b9247b5..541f71348af 100644 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java @@ -108,14 +108,12 @@ public boolean valid() { @Override public byte[] getKey() { - checkState(); checkValid(); return dbIterator.key(); } @Override public byte[] getValue() { - checkState(); checkValid(); return dbIterator.value(); } diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java index 4c635660ea6..d771716a7e8 100755 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java @@ -93,14 +93,12 @@ public boolean valid() { @Override public byte[] getKey() { - checkState(); checkValid(); return dbIterator.peekNext().getKey(); } @Override public byte[] getValue() { - checkState(); checkValid(); return dbIterator.peekNext().getValue(); } diff --git a/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java b/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java index 33e5233b551..e27990f0403 100644 --- a/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java +++ b/chainbase/src/main/java/org/tron/core/service/RewardViCalService.java @@ -3,7 +3,6 @@ import static org.tron.core.store.DelegationStore.DECIMAL_OF_VI_REWARD; import static org.tron.core.store.DelegationStore.REMARK; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Streams; import com.google.common.primitives.Bytes; import com.google.protobuf.ByteString; @@ -18,7 +17,6 @@ import java.util.stream.Collectors; import java.util.stream.LongStream; import javax.annotation.PreDestroy; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; @@ -27,9 +25,9 @@ import org.tron.common.es.ExecutorServiceManager; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; +import org.tron.common.utils.MerkleRoot; import org.tron.common.utils.Pair; import org.tron.common.utils.Sha256Hash; -import org.tron.core.capsule.utils.MerkleTree; import org.tron.core.db.common.iterator.DBIterator; import org.tron.core.db2.common.DB; import org.tron.core.store.DelegationStore; @@ -55,10 +53,11 @@ public class RewardViCalService { private volatile long lastBlockNumber = -1; - @VisibleForTesting - @Setter - private Sha256Hash rewardViRoot = Sha256Hash.wrap( - ByteString.fromHex("9debcb9924055500aaae98cdee10501c5c39d4daa75800a996f4bdda73dbccd8")); + private static final String MAIN_NET_ROOT_HEX = + "9debcb9924055500aaae98cdee10501c5c39d4daa75800a996f4bdda73dbccd8"; + + private final Sha256Hash rewardViRoot = CommonParameter.getInstance().getStorage().getDbRoot( + "reward-vi", Sha256Hash.wrap(ByteString.fromHex(MAIN_NET_ROOT_HEX))); private final CountDownLatch lock = new CountDownLatch(1); @@ -99,25 +98,30 @@ private boolean isDone() { } private void maybeRun() { - if (enableNewRewardAlgorithm()) { - if (this.newRewardCalStartCycle > 1) { - if (isDone()) { - this.clearUp(true); - logger.info("rewardViCalService is already done"); - } else { - if (lastBlockNumber == Long.MAX_VALUE // start rewardViCalService immediately - || this.getLatestBlockHeaderNumber() > lastBlockNumber) { - // checkpoint is flushed to db, so we can start rewardViCalService - startRewardCal(); - clearUp(true); + try { + if (enableNewRewardAlgorithm()) { + if (this.newRewardCalStartCycle > 1) { + if (isDone()) { + this.clearUp(true); + logger.info("rewardViCalService is already done"); } else { - logger.info("startRewardCal will run after checkpoint is flushed to db"); + if (lastBlockNumber == Long.MAX_VALUE // start rewardViCalService immediately + || this.getLatestBlockHeaderNumber() > lastBlockNumber) { + // checkpoint is flushed to db, so we can start rewardViCalService + startRewardCal(); + clearUp(true); + } else { + logger.info("startRewardCal will run after checkpoint is flushed to db"); + } } + } else { + clearUp(false); + logger.info("rewardViCalService is no need to run"); } - } else { - clearUp(false); - logger.info("rewardViCalService is no need to run"); } + } catch (Exception e) { + logger.error(" Find fatal error, program will be exited soon.", e); + System.exit(1); } } @@ -137,7 +141,7 @@ private void destroy() { public long getNewRewardAlgorithmReward(long beginCycle, long endCycle, List> votes) { - if (!rewardViStore.has(IS_DONE_KEY)) { + if (!isDone()) { logger.warn("rewardViCalService is not done, wait for it"); try { lock.await(); @@ -174,10 +178,13 @@ private void calcMerkleRoot() { .map(this::getHash) .collect(Collectors.toCollection(ArrayList::new)); - Sha256Hash rewardViRootLocal = MerkleTree.getInstance().createTree(ids).getRoot().getHash(); + Sha256Hash rewardViRootLocal = MerkleRoot.root(ids); if (!Objects.equals(rewardViRoot, rewardViRootLocal)) { - logger.warn("merkle root mismatch, expect: {}, actual: {}", - rewardViRoot, rewardViRootLocal); + logger.warn("Merkle root mismatch, expect: {}, actual: {}." + + " If you are quite sure that there is no miscalculation (not on the main network)" + + ", please configure 'storage.merkleRoot.reward-vi = {}'" + + "(for a specific network such as Nile, etc.) in config.conf to fix the hints", + rewardViRoot, rewardViRootLocal, rewardViRootLocal); } logger.info("calcMerkleRoot: {}", rewardViRootLocal); } diff --git a/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java b/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java index 23693f9bab6..bf788232640 100644 --- a/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java +++ b/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java @@ -2525,10 +2525,6 @@ public boolean useNewRewardAlgorithm() { return getNewRewardAlgorithmEffectiveCycle() != Long.MAX_VALUE; } - public boolean useNewRewardAlgorithmFromStart() { - return getNewRewardAlgorithmEffectiveCycle() == 1; - } - public void saveNewRewardAlgorithmEffectiveCycle() { if (getNewRewardAlgorithmEffectiveCycle() == Long.MAX_VALUE) { long currentCycle = getCurrentCycleNumber(); @@ -2839,19 +2835,8 @@ public boolean supportMaxDelegateLockPeriod() { getUnfreezeDelayDays() > 0; } - /** - * @require NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE != Long.MAX_VALUE - * @require NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE > 1 - */ public void saveAllowOldRewardOpt(long allowOldRewardOpt) { - if (useNewRewardAlgorithm()) { - if (useNewRewardAlgorithmFromStart()) { - throw new IllegalStateException("no need old reward opt, ALLOW_NEW_REWARD from start"); - } - this.put(ALLOW_OLD_REWARD_OPT, new BytesCapsule(ByteArray.fromLong(allowOldRewardOpt))); - } else { - throw new IllegalStateException("not support old reward opt, ALLOW_NEW_REWARD not set"); - } + this.put(ALLOW_OLD_REWARD_OPT, new BytesCapsule(ByteArray.fromLong(allowOldRewardOpt))); } public boolean allowOldRewardOpt() { diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 7e85771000e..87ab6fae0a3 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -62,8 +62,6 @@ public static class Histogram { public static final String MESSAGE_PROCESS_LATENCY = "tron:message_process_latency_seconds"; public static final String BLOCK_FETCH_LATENCY = "tron:block_fetch_latency_seconds"; public static final String BLOCK_RECEIVE_DELAY = "tron:block_receive_delay_seconds"; - public static final String DO_REWARD_CAL_DELAY = "tron:do_reward_cal_seconds"; - private Histogram() { throw new IllegalStateException("Histogram"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java b/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java index d776474c92e..556db10feb5 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java @@ -48,8 +48,6 @@ public class MetricsHistogram { init(MetricKeys.Histogram.BLOCK_FETCH_LATENCY, "fetch block latency."); init(MetricKeys.Histogram.BLOCK_RECEIVE_DELAY, "receive block delay time, receiveTime - blockTime."); - init(MetricKeys.Histogram.DO_REWARD_CAL_DELAY, - "do reward cal delay time.", "depth"); } private MetricsHistogram() { diff --git a/common/src/main/java/org/tron/common/utils/MerkleRoot.java b/common/src/main/java/org/tron/common/utils/MerkleRoot.java new file mode 100644 index 00000000000..ccd8905b6c5 --- /dev/null +++ b/common/src/main/java/org/tron/common/utils/MerkleRoot.java @@ -0,0 +1,68 @@ +package org.tron.common.utils; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import lombok.Getter; + +public class MerkleRoot { + + private MerkleRoot() { + + } + + public static Sha256Hash root(List hashList) { + List leaves = createLeaves(hashList); + while (leaves.size() > 1) { + leaves = createParentLeaves(leaves); + } + return leaves.isEmpty() ? Sha256Hash.ZERO_HASH : leaves.get(0).hash; + } + + private static List createParentLeaves(List leaves) { + int step = 2; + int len = leaves.size(); + return IntStream.iterate(0, i -> i + step) + .limit(len) + .filter(i -> i < len) + .mapToObj(i -> { + Leaf right = i + 1 < len ? leaves.get(i + 1) : null; + return createLeaf(leaves.get(i), right); + }).collect(Collectors.toList()); + } + + private static List createLeaves(List hashList) { + int step = 2; + int len = hashList.size(); + return IntStream.iterate(0, i -> i + step) + .limit(len) + .filter(i -> i < len) + .mapToObj(i -> { + Leaf right = i + 1 < len ? createLeaf(hashList.get(i + 1)) : null; + return createLeaf(createLeaf(hashList.get(i)), right); + }).collect(Collectors.toList()); + } + + private static Leaf createLeaf(Leaf left, Leaf right) { + Leaf leaf = new Leaf(); + leaf.hash = right == null ? left.hash : computeHash(left.hash, right.hash); + return leaf; + } + + private static Leaf createLeaf(Sha256Hash hash) { + Leaf leaf = new Leaf(); + leaf.hash = hash; + return leaf; + } + + private static Sha256Hash computeHash(Sha256Hash leftHash, Sha256Hash rightHash) { + return Sha256Hash.of(true, + leftHash.getByteString().concat(rightHash.getByteString()).toByteArray()); + } + + @Getter + private static class Leaf { + + private Sha256Hash hash; + } +} diff --git a/common/src/main/java/org/tron/core/config/args/Storage.java b/common/src/main/java/org/tron/core/config/args/Storage.java index 22dc239c167..9cf6eb6bab1 100644 --- a/common/src/main/java/org/tron/core/config/args/Storage.java +++ b/common/src/main/java/org/tron/core/config/args/Storage.java @@ -16,6 +16,7 @@ package org.tron.core.config.args; import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; import com.typesafe.config.Config; import com.typesafe.config.ConfigObject; import java.io.File; @@ -32,6 +33,7 @@ import org.tron.common.utils.DbOptionalsUtils; import org.tron.common.utils.FileUtil; import org.tron.common.utils.Property; +import org.tron.common.utils.Sha256Hash; /** * Custom storage configurations @@ -79,6 +81,8 @@ public class Storage { private static final String CACHE_STRATEGIES = "storage.cache.strategies"; public static final String TX_CACHE_INIT_OPTIMIZATION = "storage.txCache.initOptimization"; + private static final String MERKLE_ROOT = "storage.merkleRoot"; + /** * Default values of directory */ @@ -163,6 +167,9 @@ public class Storage { @Getter private Map propertyMap; + // db root + private final Map dbRoots = Maps.newConcurrentMap(); + public static String getDbEngineFromConfig(final Config config) { return config.hasPath(DB_ENGINE_CONFIG_KEY) ? config.getString(DB_ENGINE_CONFIG_KEY) : DEFAULT_DB_ENGINE; @@ -258,6 +265,18 @@ public String getCacheStrategy(CacheType dbName) { return this.cacheStrategies.getOrDefault(dbName, CacheStrategies.getCacheStrategy(dbName)); } + public Sha256Hash getDbRoot(String dbName, Sha256Hash defaultV) { + return this.dbRoots.getOrDefault(dbName, defaultV); + } + + public void setDbRoots(Config config) { + if (config.hasPath(MERKLE_ROOT)) { + config.getConfig(MERKLE_ROOT).resolve().entrySet().forEach(c -> + this.dbRoots.put(c.getKey(), Sha256Hash.wrap( + ByteString.fromHex(c.getValue().unwrapped().toString())))); + } + } + private Property createProperty(final ConfigObject conf) { Property property = new Property(); diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 9694668af01..610e491c100 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -530,6 +530,7 @@ public static void setParam(final String[] args, final String confFileName) { PARAMETER.storage.setDefaultDbOptions(config); PARAMETER.storage.setPropertyMapFromConfig(config); PARAMETER.storage.setCacheStrategies(config); + PARAMETER.storage.setDbRoots(config); PARAMETER.seedNode = new SeedNode(); PARAMETER.seedNode.setAddressList(loadSeeds(config)); @@ -1192,9 +1193,17 @@ public static void setParam(final String[] args, final String confFileName) { config.hasPath(Constant.MAX_UNSOLIDIFIED_BLOCKS) ? config .getInt(Constant.MAX_UNSOLIDIFIED_BLOCKS) : 1000; - PARAMETER.allowOldRewardOpt = - config.hasPath(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) ? config - .getInt(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) : 0; + long allowOldRewardOpt = config.hasPath(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) ? config + .getInt(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) : 0; + if (allowOldRewardOpt == 1 && PARAMETER.allowNewRewardAlgorithm != 1 + && PARAMETER.allowNewReward != 1 && PARAMETER.allowTvmVote != 1) { + throw new IllegalArgumentException( + "At least one of the following proposals is required to be opened first: " + + "committee.allowNewRewardAlgorithm = 1" + + " or committee.allowNewReward = 1" + + " or committee.allowTvmVote = 1."); + } + PARAMETER.allowOldRewardOpt = allowOldRewardOpt; logConfig(); } diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index f0479a29cb2..5fe32b82880 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -97,6 +97,13 @@ storage { # txCache.estimatedTransactions = 1000 # if true, transaction cache initialization will be faster. default false # txCache.initOptimization = true + + # data root setting, for check data, currently, only reward-vi is used. + + # merkleRoot = { + # reward-vi = 9debcb9924055500aaae98cdee10501c5c39d4daa75800a996f4bdda73dbccd8 // main-net, Sha256Hash, hexString + # } + } node.discovery = { diff --git a/framework/src/test/java/org/tron/core/actuator/utils/ProposalUtilTest.java b/framework/src/test/java/org/tron/core/actuator/utils/ProposalUtilTest.java index 8866540a028..8e2b7647967 100644 --- a/framework/src/test/java/org/tron/core/actuator/utils/ProposalUtilTest.java +++ b/framework/src/test/java/org/tron/core/actuator/utils/ProposalUtilTest.java @@ -364,22 +364,10 @@ public void validateCheck() { Assert.fail(); } catch (ContractValidateException e) { Assert.assertEquals( - "[ALLOW_NEW_REWARD] proposal must be approved " + "[ALLOW_NEW_REWARD] or [ALLOW_TVM_VOTE] proposal must be approved " + "before [ALLOW_OLD_REWARD_OPT] can be proposed", e.getMessage()); } - dynamicPropertiesStore.saveCurrentCycleNumber(0); - dynamicPropertiesStore.saveNewRewardAlgorithmEffectiveCycle(); - dynamicPropertiesStore.saveAllowNewReward(1); - try { - ProposalUtil.validator(dynamicPropertiesStore, forkUtils, - ProposalType.ALLOW_OLD_REWARD_OPT.getCode(), 1); - Assert.fail(); - } catch (ContractValidateException e) { - Assert.assertEquals( - "no need old reward opt, ALLOW_NEW_REWARD from start cycle 1", - e.getMessage()); - } dynamicPropertiesStore.put("NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE".getBytes(), new BytesCapsule(ByteArray.fromLong(4000))); dynamicPropertiesStore.saveAllowOldRewardOpt(1); diff --git a/framework/src/test/java/org/tron/core/capsule/utils/MerkleTreeTest.java b/framework/src/test/java/org/tron/core/capsule/utils/MerkleTreeTest.java index 4e3273203c8..3662fb524b8 100644 --- a/framework/src/test/java/org/tron/core/capsule/utils/MerkleTreeTest.java +++ b/framework/src/test/java/org/tron/core/capsule/utils/MerkleTreeTest.java @@ -1,18 +1,28 @@ package org.tron.core.capsule.utils; +import com.google.protobuf.ByteString; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; +import org.tron.common.utils.MerkleRoot; import org.tron.common.utils.Sha256Hash; import org.tron.core.capsule.utils.MerkleTree.Leaf; @Slf4j public class MerkleTreeTest { + @Rule + public ExpectedException exception = ExpectedException.none(); + private static List getHash(int hashNum) { List hashList = new ArrayList(); for (int i = 0; i < hashNum; i++) { @@ -173,4 +183,27 @@ public void testAnyHashNum() { pareTree(root, hashList, maxRank, 0, 0); } } -} \ No newline at end of file + + @Test + public void testConcurrent() { + Sha256Hash root1 = Sha256Hash.wrap( + ByteString.fromHex("6cb38b4f493db8bacf26123cd4253bbfc530c708b97b3747e782f64097c3c482")); + Sha256Hash root2 = Sha256Hash.wrap( + ByteString.fromHex("4bfc60ea3de4f5d1476f839874df0aba38eec4e524d6fa63f5b19c4bf527eaf3")); + List list1 = IntStream.range(0, 10000).mapToObj(i -> + Sha256Hash.of(true, ("byte1-" + i).getBytes(StandardCharsets.UTF_8))) + .collect(Collectors.toList()); + List list2 = IntStream.range(0, 10000).mapToObj(i -> + Sha256Hash.of(true, ("byte2-" + i).getBytes(StandardCharsets.UTF_8))) + .collect(Collectors.toList()); + Assert.assertEquals(root1, MerkleTree.getInstance().createTree(list1).getRoot().getHash()); + Assert.assertEquals(root2, MerkleTree.getInstance().createTree(list2).getRoot().getHash()); + Assert.assertEquals(root1, MerkleRoot.root(list1)); + Assert.assertEquals(root2, MerkleRoot.root(list2)); + exception.expect(ArrayIndexOutOfBoundsException.class); + IntStream.range(0, 1000).parallel().forEach(i -> Assert.assertEquals( + MerkleTree.getInstance().createTree(i % 2 == 0 ? list1 : list2).getRoot().getHash(), + MerkleRoot.root(i % 2 == 0 ? list1 : list2)) + ); + } +} diff --git a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java index 59b684b338f..bd3d34e77f7 100644 --- a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java +++ b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java @@ -25,7 +25,9 @@ import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.tron.common.args.GenesisBlock; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; @@ -41,6 +43,9 @@ public class ArgsTest { private String address; private LocalWitnesses localWitnesses; + @Rule + public ExpectedException thrown = ExpectedException.none(); + @After public void destroy() { Args.clearParam(); @@ -138,5 +143,11 @@ public void testIpFromLibP2p() Assert.assertNotEquals(configuredExternalIp, parameter.getNodeExternalIp()); } + + @Test + public void testOldRewardOpt() { + thrown.expect(IllegalArgumentException.class); + Args.setParam(new String[] {"-w"}, "args-test.conf"); + } } diff --git a/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java b/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java index 00e0799cf11..0082c8728da 100644 --- a/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java +++ b/framework/src/test/java/org/tron/core/services/ComputeRewardTest.java @@ -17,13 +17,16 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.contrib.java.lang.system.ExpectedSystemExit; import org.junit.rules.TemporaryFolder; import org.tron.common.application.TronApplicationContext; import org.tron.common.error.TronDBException; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.utils.ByteArray; +import org.tron.common.utils.ReflectUtils; import org.tron.core.Constant; import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.BytesCapsule; import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; @@ -32,6 +35,7 @@ import org.tron.core.store.AccountStore; import org.tron.core.store.DelegationStore; import org.tron.core.store.DynamicPropertiesStore; +import org.tron.core.store.RewardViStore; import org.tron.core.store.WitnessStore; import org.tron.protos.Protocol; @@ -106,9 +110,13 @@ public class ComputeRewardTest { private static RewardViCalService rewardViCalService; private static WitnessStore witnessStore; private static MortgageService mortgageService; + private static RewardViStore rewardViStore; @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule + public final ExpectedSystemExit exit = ExpectedSystemExit.none(); + @After public void destroy() { context.destroy(); @@ -129,6 +137,7 @@ public void init() throws IOException { rewardViCalService = context.getBean(RewardViCalService.class); witnessStore = context.getBean(WitnessStore.class); mortgageService = context.getBean(MortgageService.class); + rewardViStore = context.getBean(RewardViStore.class); setUp(); } @@ -254,7 +263,32 @@ private void setUp() { @Test public void query() { + exit.expectSystemExitWithStatus(1); Assert.assertEquals(3189, mortgageService.queryReward(OWNER_ADDRESS)); + // mock root is error + rewardViStore.put("test".getBytes(), "test".getBytes()); + ReflectUtils.invokeMethod(rewardViCalService,"maybeRun"); + + // mock no need + propertiesStore.saveCurrentCycleNumber(0); + // reset + propertiesStore.put("NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE".getBytes(), + new BytesCapsule(ByteArray.fromLong(Long.MAX_VALUE))); + // set + propertiesStore.saveNewRewardAlgorithmEffectiveCycle(); + ReflectUtils.invokeMethod(rewardViCalService,"maybeRun"); + + // mock maybeRun exception + propertiesStore.saveCurrentCycleNumber(4); + // reset + propertiesStore.put("NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE".getBytes(), + new BytesCapsule(ByteArray.fromLong(Long.MAX_VALUE))); + // set + propertiesStore.saveNewRewardAlgorithmEffectiveCycle(); + propertiesStore.saveCurrentCycleNumber(5); + rewardViStore.close(); + ReflectUtils.invokeMethod(rewardViCalService,"maybeRun"); + } static class Vote { diff --git a/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java b/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java index e248af647c3..0ba32b27f2e 100644 --- a/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java +++ b/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java @@ -1,6 +1,5 @@ package org.tron.core.services; -import static org.tron.core.utils.ProposalUtil.ProposalType.ALLOW_OLD_REWARD_OPT; import static org.tron.core.utils.ProposalUtil.ProposalType.ENERGY_FEE; import static org.tron.core.utils.ProposalUtil.ProposalType.TRANSACTION_FEE; import static org.tron.core.utils.ProposalUtil.ProposalType.WITNESS_127_PAY_PER_BLOCK; @@ -11,9 +10,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.tron.common.BaseTest; import org.tron.core.Constant; import org.tron.core.capsule.ProposalCapsule; @@ -25,9 +22,6 @@ @Slf4j public class ProposalServiceTest extends BaseTest { - @Rule - public ExpectedException thrown = ExpectedException.none(); - private static boolean init; @BeforeClass @@ -69,9 +63,6 @@ public void test() { proposal = Proposal.newBuilder().putParameters(proposalType.getCode(), 1).build(); } proposalCapsule = new ProposalCapsule(proposal); - if (proposalType == ALLOW_OLD_REWARD_OPT) { - thrown.expect(IllegalStateException.class); - } result = ProposalService.process(dbManager, proposalCapsule); Assert.assertTrue(result); } diff --git a/framework/src/test/resources/args-test.conf b/framework/src/test/resources/args-test.conf new file mode 100644 index 00000000000..91913dfe32e --- /dev/null +++ b/framework/src/test/resources/args-test.conf @@ -0,0 +1,224 @@ +net { + // type = mainnet + type = testnet +} + + +storage { + # Directory for storing persistent data + + db.engine = "LEVELDB" + db.directory = "database", + index.directory = "index", + + # You can custom these 14 databases' configs: + + # account, account-index, asset-issue, block, block-index, + # block_KDB, peers, properties, recent-block, trans, + # utxo, votes, witness, witness_schedule. + + # Otherwise, db configs will remain defualt and data will be stored in + # the path of "output-directory" or which is set by "-d" ("--output-directory"). + + # Attention: name is a required field that must be set !!! + default = { + maxOpenFiles = 50 + } + defaultM = { + maxOpenFiles = 500 + } + defaultL = { + maxOpenFiles = 1000 + } + properties = [ + { + name = "account", + path = "storage_directory_test", + createIfMissing = true, + paranoidChecks = true, + verifyChecksums = true, + compressionType = 1, // compressed with snappy + blockSize = 4096, // 4 KB = 4 * 1024 B + writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + maxOpenFiles = 100 + }, + { + name = "account-index", + path = "storage_directory_test", + createIfMissing = true, + paranoidChecks = true, + verifyChecksums = true, + compressionType = 1, // compressed with snappy + blockSize = 4096, // 4 KB = 4 * 1024 B + writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + maxOpenFiles = 100 + }, + { # only for unit test + name = "test_name", + path = "test_path", + createIfMissing = false, + paranoidChecks = false, + verifyChecksums = false, + compressionType = 1, + blockSize = 2, + writeBufferSize = 3, + cacheSize = 4, + maxOpenFiles = 5 + }, + ] + + needToUpdateAsset = false + +} + +node.discovery = { + enable = true + persist = true + external.ip = "46.168.1.1" +} + +node { + + trustNode = "127.0.0.1:50051" + + listen.port = 18888 + + connection.timeout = 2 + + active = [] + + maxConnections = 30 + minConnections = 8 + minActiveConnections = 3 + + p2p { + version = 43 # 43: testnet; 101: debug + } + + rpc { + port = 50051 + } + +} + +sync { + node.count = 30 +} + +seed.node = { + ip.list = [ + ] +} + +genesis.block = { + # Reserve balance + assets = [ + { + accountName = "Devaccount" + accountType = "AssetIssue" + address = "27d3byPxZXKQWfXX7sJvemJJuv5M65F3vjS" + balance = "10000000000000000" + }, + { + accountName = "Zion" + accountType = "AssetIssue" + address = "27fXgQ46DcjEsZ444tjZPKULcxiUfDrDjqj" + balance = "15000000000000000" + }, + { + accountName = "Sun" + accountType = "AssetIssue" + address = "27SWXcHuQgFf9uv49FknBBBYBaH3DUk4JPx" + balance = "10000000000000000" + }, + { + accountName = "Blackhole" + accountType = "AssetIssue" + address = "27WtBq2KoSy5v8VnVZBZHHJcDuWNiSgjbE3" + balance = "-9223372036854775808" + } + ] + + witnesses = [ + { + address: 27Ssb1WE8FArwJVRRb8Dwy3ssVGuLY8L3S1 + url = "http://Mercury.org", + voteCount = 105 + }, + { + address: 27anh4TDZJGYpsn4BjXzb7uEArNALxwiZZW + url = "http://Venus.org", + voteCount = 104 + }, + { + address: 27Wkfa5iEJtsKAKdDzSmF1b2gDm5s49kvdZ + url = "http://Earth.org", + voteCount = 103 + }, + { + address: 27bqKYX9Bgv7dgTY7xBw5SUHZ8EGaPSikjx + url = "http://Mars.org", + voteCount = 102 + }, + { + address: 27fASUY6qKtsaAEPz6QxhZac2KYVz2ZRTXW + url = "http://Jupiter.org", + voteCount = 101 + }, + { + address: 27Q3RSbiqm59VXcF8shQWHKbyztfso5FwvP + url = "http://Saturn.org", + voteCount = 100 + }, + { + address: 27YkUVSuvCK3K84DbnFnxYUxozpi793PTqZ + url = "http://Uranus.org", + voteCount = 99 + }, + { + address: 27kdTBTDJ16hK3Xqr8PpCuQJmje1b94CDJU + url = "http://Neptune.org", + voteCount = 98 + }, + { + address: 27mw9UpRy7inTMQ5kUzsdTc2QZ6KvtCX4uB + url = "http://Pluto.org", + voteCount = 97 + }, + { + address: 27QzC4PeQZJ2kFMUXiCo4S8dx3VWN5U9xcg + url = "http://Altair.org", + voteCount = 96 + }, + { + address: 27VZHn9PFZwNh7o2EporxmLkpe157iWZVkh + url = "http://AlphaLyrae.org", + voteCount = 95 + } + ] + + timestamp = "0" #2017-8-26 12:00:00 + + parentHash = "0x0000000000000000000000000000000000000000000000000000000000000000" +} + + +localwitness = [ + +] + +block = { + needSyncCheck = true # first node : false, other : true +} + +vm = { + supportConstant = true + minTimeRatio = 0.0 + maxTimeRatio = 5.0 +} +committee = { + allowCreationOfContracts = 1 //mainnet:0 (reset by committee),test:1 + allowOldRewardOpt = 1 +} diff --git a/framework/src/test/resources/config-test-storagetest.conf b/framework/src/test/resources/config-test-storagetest.conf index d2dc27b795a..25127cdab91 100644 --- a/framework/src/test/resources/config-test-storagetest.conf +++ b/framework/src/test/resources/config-test-storagetest.conf @@ -284,4 +284,6 @@ vm = { } committee = { allowCreationOfContracts = 1 //mainnet:0 (reset by committee),test:1 + allowOldRewardOpt = 1 + allowNewRewardAlgorithm = 1 } diff --git a/framework/src/test/resources/config-test.conf b/framework/src/test/resources/config-test.conf index 68a48ca64b2..db24bb2a8a0 100644 --- a/framework/src/test/resources/config-test.conf +++ b/framework/src/test/resources/config-test.conf @@ -69,6 +69,12 @@ storage { # if true, transaction cache initialization will be faster. default false txCache.initOptimization = true + # data root setting, for check data, currently, only reward-vi is used. + + merkleRoot = { + reward-vi = e0ebe2f3243391ed674dff816a07f589a3279420d6d88bc823b6a9d5778337ce + } + } node.discovery = { diff --git a/plugins/src/main/java/org/tron/plugins/Db.java b/plugins/src/main/java/org/tron/plugins/Db.java index c67c838a3d6..84654dca934 100644 --- a/plugins/src/main/java/org/tron/plugins/Db.java +++ b/plugins/src/main/java/org/tron/plugins/Db.java @@ -11,7 +11,8 @@ DbArchive.class, DbConvert.class, DbLite.class, - DbCopy.class + DbCopy.class, + DbRoot.class }, commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n" ) diff --git a/plugins/src/main/java/org/tron/plugins/DbRoot.java b/plugins/src/main/java/org/tron/plugins/DbRoot.java new file mode 100644 index 00000000000..69555c16d33 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/DbRoot.java @@ -0,0 +1,99 @@ +package org.tron.plugins; + +import com.google.common.collect.Streams; +import com.google.common.primitives.Bytes; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import me.tongfei.progressbar.ProgressBar; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.MerkleRoot; +import org.tron.plugins.utils.Sha256Hash; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; +import org.tron.plugins.utils.db.DbTool; +import picocli.CommandLine; + +@Slf4j(topic = "db-root") +@CommandLine.Command(name = "root", + description = "compute merkle root for tiny db. NOTE: large db may GC overhead limit exceeded.", + exitCodeListHeading = "Exit Codes:%n", + exitCodeList = { + "0:Successful", + "n:query failed,please check toolkit.log"}) +public class DbRoot implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + @CommandLine.Parameters(index = "0", defaultValue = "output-directory/database", + description = "Input path. Default: ${DEFAULT-VALUE}") + private Path db; + + @CommandLine.Option(names = { "--db"}, + description = "db name for show root") + private List dbs; + + @CommandLine.Option(names = {"-h", "--help"}, help = true, description = "display a help message") + private boolean help; + + + @Override + public Integer call() throws Exception { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + if (!db.toFile().exists()) { + logger.info(" {} does not exist.", db); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText(String.format("%s does not exist.", db))); + return 404; + } + + // remove not exit + if (dbs != null) { + dbs.removeIf(s -> !Paths.get(db.toString(), s).toFile().exists()); + } + + if (dbs == null || dbs.isEmpty()) { + logger.info("Specify at least one exit database: --db dbName."); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText("Specify at least one exit database: --db dbName.")); + return 404; + } + if (dbs.size() > 1) { + ProgressBar.wrap(dbs.stream(), "root task").parallel().forEach(this::calcMerkleRoot); + } else { + calcMerkleRoot(dbs.get(0)); + } + spec.commandLine().getOut().println("root task done."); + return 0; + } + + private void calcMerkleRoot(String name) { + try (DBInterface database = DbTool.getDB(this.db, name)) { + DBIterator iterator = database.iterator(); + iterator.seekToFirst(); + ArrayList ids = Streams.stream(iterator) + .map(this::getHash) + .collect(Collectors.toCollection(ArrayList::new)); + Sha256Hash root = MerkleRoot.root(ids); + logger.info("db: {},root: {}", database.getName(), root); + spec.commandLine().getOut().println(String.format("db: %s,root: %s", + database.getName(), root)); + } catch (RocksDBException | IOException e) { + logger.error("calc db {} fail", name, e); + } + } + + private Sha256Hash getHash(Map.Entry entry) { + return Sha256Hash.of(true, + Bytes.concat(entry.getKey(), entry.getValue())); + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/MerkleRoot.java b/plugins/src/main/java/org/tron/plugins/utils/MerkleRoot.java new file mode 100644 index 00000000000..055f5dcdee0 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/MerkleRoot.java @@ -0,0 +1,68 @@ +package org.tron.plugins.utils; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import lombok.Getter; + +public class MerkleRoot { + + private MerkleRoot() { + + } + + public static Sha256Hash root(List hashList) { + List leaves = createLeaves(hashList); + while (leaves.size() > 1) { + leaves = createParentLeaves(leaves); + } + return leaves.isEmpty() ? Sha256Hash.ZERO_HASH : leaves.get(0).hash; + } + + private static List createParentLeaves(List leaves) { + int step = 2; + int len = leaves.size(); + return IntStream.iterate(0, i -> i + step) + .limit(len) + .filter(i -> i < len) + .mapToObj(i -> { + Leaf right = i + 1 < len ? leaves.get(i + 1) : null; + return createLeaf(leaves.get(i), right); + }).collect(Collectors.toList()); + } + + private static List createLeaves(List hashList) { + int step = 2; + int len = hashList.size(); + return IntStream.iterate(0, i -> i + step) + .limit(len) + .filter(i -> i < len) + .mapToObj(i -> { + Leaf right = i + 1 < len ? createLeaf(hashList.get(i + 1)) : null; + return createLeaf(createLeaf(hashList.get(i)), right); + }).collect(Collectors.toList()); + } + + private static Leaf createLeaf(Leaf left, Leaf right) { + Leaf leaf = new Leaf(); + leaf.hash = right == null ? left.hash : computeHash(left.hash, right.hash); + return leaf; + } + + private static Leaf createLeaf(Sha256Hash hash) { + Leaf leaf = new Leaf(); + leaf.hash = hash; + return leaf; + } + + private static Sha256Hash computeHash(Sha256Hash leftHash, Sha256Hash rightHash) { + return Sha256Hash.of(true, + leftHash.getByteString().concat(rightHash.getByteString()).toByteArray()); + } + + @Getter + private static class Leaf { + + private Sha256Hash hash; + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java b/plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java index 5fe80601b66..67e6e64ea79 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java +++ b/plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java @@ -21,6 +21,7 @@ import com.google.common.io.ByteStreams; import com.google.common.primitives.Ints; +import com.google.protobuf.ByteString; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -39,7 +40,7 @@ public class Sha256Hash implements Serializable, Comparable { public static final int LENGTH = 32; // bytes - + public static final Sha256Hash ZERO_HASH = wrap(new byte[LENGTH]); private final byte[] bytes; /** @@ -184,6 +185,13 @@ public byte[] getBytes() { return bytes; } + /** + * For pb return ByteString. + */ + public ByteString getByteString() { + return ByteString.copyFrom(bytes); + } + @Override public int compareTo(final Sha256Hash other) { diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java b/plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java index b0f7c58c587..513e021c83c 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java +++ b/plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java @@ -18,4 +18,6 @@ public interface DBInterface extends Closeable { void close() throws IOException; + String getName(); + } diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java b/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java index 6e8df4c59fb..429025e8f8b 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java +++ b/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java @@ -23,7 +23,7 @@ public class DbTool { private static final Map dbMap = Maps.newConcurrentMap(); - enum DbType { + public enum DbType { LevelDB, RocksDB } @@ -83,11 +83,11 @@ public static DBInterface getDB(String sourceDir, String dbName, DbType type) DBInterface db; switch (type) { case LevelDB: - db = openLevelDb(path); + db = openLevelDb(path, dbName); dbMap.put(path.toString(), db); break; case RocksDB: - db = openRocksDb(path); + db = openRocksDb(path, dbName); dbMap.put(path.toString(), db); break; default: @@ -114,9 +114,9 @@ public static DBInterface getDB(Path sourceDir, String dbName) DbType type = getDbType(sourceDir.toString(), dbName); switch (type) { case LevelDB: - return openLevelDb(path); + return openLevelDb(path, dbName); case RocksDB: - return openRocksDb(path); + return openRocksDb(path, dbName); default: throw new IllegalStateException("Unexpected value: " + type); } @@ -175,12 +175,12 @@ private static DbType getDbType(String sourceDir, String dbName) { } } - private static LevelDBImpl openLevelDb(Path db) throws IOException { - return new LevelDBImpl(DBUtils.newLevelDb(db)); + private static LevelDBImpl openLevelDb(Path db, String name) throws IOException { + return new LevelDBImpl(DBUtils.newLevelDb(db), name); } - private static RocksDBImpl openRocksDb(Path db) throws RocksDBException { - return new RocksDBImpl(DBUtils.newRocksDb(db)); + private static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { + return new RocksDBImpl(DBUtils.newRocksDb(db), name); } diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java b/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java index 3a89e77af40..511f4dfd5b4 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java +++ b/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java @@ -2,6 +2,7 @@ import com.google.common.collect.Streams; import java.io.IOException; +import lombok.Getter; import org.iq80.leveldb.DB; import org.iq80.leveldb.ReadOptions; @@ -10,8 +11,12 @@ public class LevelDBImpl implements DBInterface { private DB leveldb; - public LevelDBImpl(DB leveldb) { + @Getter + private final String name; + + public LevelDBImpl(DB leveldb, String name) { this.leveldb = leveldb; + this.name = name; } @Override diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java b/plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java index 88b25b0413f..50957bbe61b 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java +++ b/plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java @@ -1,6 +1,7 @@ package org.tron.plugins.utils.db; import java.io.IOException; +import lombok.Getter; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; @@ -8,8 +9,12 @@ public class RocksDBImpl implements DBInterface { private org.rocksdb.RocksDB rocksDB; - public RocksDBImpl(org.rocksdb.RocksDB rocksDB) { + @Getter + private final String name; + + public RocksDBImpl(org.rocksdb.RocksDB rocksDB, String name) { this.rocksDB = rocksDB; + this.name = name; } @Override diff --git a/plugins/src/test/java/org/tron/plugins/DbRootTest.java b/plugins/src/test/java/org/tron/plugins/DbRootTest.java new file mode 100644 index 00000000000..4b2c6c7d373 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbRootTest.java @@ -0,0 +1,82 @@ +package org.tron.plugins; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; +import org.tron.plugins.utils.db.LevelDBImpl; +import picocli.CommandLine; + +@Slf4j +public class DbRootTest { + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + + CommandLine cli = new CommandLine(new Toolkit()); + + private static final String NORMAL_DB = "normal"; + private static final String EMPTY_DB = "empty"; + private static final String ERROR_DB = "error"; + + @Test + public void testRoot() throws IOException, RocksDBException { + + File file = folder.newFolder(); + + File database = Paths.get(file.getPath(),"database").toFile(); + Assert.assertTrue(database.mkdirs()); + + + try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, DbTool.DbType.LevelDB); + DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, DbTool.DbType.RocksDB)) { + for (int i = 0; i < 10; i++) { + normal.put(("" + i).getBytes(), (NORMAL_DB + "-" + i).getBytes()); + } + } + + String[] args = new String[] {"db", "root", database.toString(), + "--db", NORMAL_DB, "--db", EMPTY_DB}; + Assert.assertEquals(0, cli.execute(args)); + args = new String[] {"db", "root", database.toString(), + "--db", NORMAL_DB}; + Assert.assertEquals(0, cli.execute(args)); + args = new String[] {"db", "root", database.toString(), + "--db", EMPTY_DB}; + Assert.assertEquals(0, cli.execute(args)); + + try (DBInterface errorDb = new LevelDBImpl( + DBUtils.newLevelDb(Paths.get(database.toString(), ERROR_DB)), ERROR_DB)) { + for (int i = 0; i < 10; i++) { + errorDb.put(("" + i).getBytes(), (ERROR_DB + "-" + i).getBytes()); + } + args = new String[] {"db", "root", database.toString(), "--db", ERROR_DB}; + Assert.assertEquals(0, cli.execute(args)); + } + + } + + @Test + public void testHelp() { + String[] args = new String[] {"db", "root", "-h"}; + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testEmpty() throws IOException { + File file = folder.newFolder(); + File database = Paths.get(file.getPath(),"database").toFile(); + String[] args = new String[] {"db", "root", database.toString()}; + Assert.assertEquals(404, cli.execute(args)); + Assert.assertTrue(database.mkdirs()); + Assert.assertEquals(404, cli.execute(args)); + } +} From 074e0d69b6b87ea1d070758b29f80dd6917cdd15 Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Wed, 6 Mar 2024 12:37:55 +0800 Subject: [PATCH 56/61] hotfix(net):throw exception if scalingFactor <=0 (#5746) --- .../tron/core/capsule/TransactionCapsule.java | 1 - .../src/main/java/org/tron/core/Wallet.java | 17 +++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java b/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java index 5d5a5d5b462..a588b28c748 100755 --- a/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java +++ b/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java @@ -738,7 +738,6 @@ public Transaction getInstance() { @Override public String toString() { StringBuilder toStringBuff = new StringBuilder(); - toStringBuff.setLength(0); toStringBuff.append("TransactionCapsule \n[ "); toStringBuff.append("hash=").append(getTransactionId()).append("\n"); diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index 6e5c17417b7..96af6fc7476 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -3862,12 +3862,11 @@ private boolean isShieldedTRC20NoteSpent(GrpcAPI.Note note, long pos, byte[] ak, TransactionExtention.Builder trxExtBuilder = TransactionExtention.newBuilder(); Return.Builder retBuilder = Return.newBuilder(); TransactionExtention trxExt; - Transaction trx; try { TransactionCapsule trxCap = createTransactionCapsule(trigger, ContractType.TriggerSmartContract); - trx = triggerConstantContract(trigger, trxCap, trxExtBuilder, retBuilder); + Transaction trx = triggerConstantContract(trigger, trxCap, trxExtBuilder, retBuilder); retBuilder.setResult(true).setCode(response_code.SUCCESS); trxExtBuilder.setTransaction(trx); @@ -3890,10 +3889,10 @@ private boolean isShieldedTRC20NoteSpent(GrpcAPI.Note note, long pos, byte[] ak, logger.warn("unknown exception caught: " + e.getMessage(), e); } finally { trxExt = trxExtBuilder.build(); - trx = trxExt.getTransaction(); } - if (code.SUCESS == trx.getRet(0).getRet()) { + String code = trxExt.getResult().getCode().toString(); + if ("SUCCESS".equals(code)) { List list = trxExt.getConstantResultList(); byte[] listBytes = new byte[0]; for (ByteString bs : list) { @@ -4086,6 +4085,9 @@ private long[] checkPublicAmount(byte[] address, BigInteger fromAmount, BigInteg } catch (ContractExeException e) { throw new ContractExeException("Get shielded contract scalingFactor failed"); } + if (scalingFactor.compareTo(BigInteger.ZERO) <= 0) { + throw new ContractValidateException("scalingFactor must be positive"); + } // fromAmount and toAmount must be a multiple of scalingFactor if (!(fromAmount.mod(scalingFactor).equals(BigInteger.ZERO) @@ -4127,12 +4129,11 @@ public byte[] getShieldedContractScalingFactor(byte[] contractAddress) TransactionExtention.Builder trxExtBuilder = TransactionExtention.newBuilder(); Return.Builder retBuilder = Return.newBuilder(); TransactionExtention trxExt; - Transaction trx; try { TransactionCapsule trxCap = createTransactionCapsule(trigger, ContractType.TriggerSmartContract); - trx = triggerConstantContract(trigger, trxCap, trxExtBuilder, retBuilder); + Transaction trx = triggerConstantContract(trigger, trxCap, trxExtBuilder, retBuilder); retBuilder.setResult(true).setCode(response_code.SUCCESS); trxExtBuilder.setTransaction(trx); @@ -4155,10 +4156,10 @@ public byte[] getShieldedContractScalingFactor(byte[] contractAddress) logger.warn("Unknown exception caught: " + e.getMessage(), e); } finally { trxExt = trxExtBuilder.build(); - trx = trxExt.getTransaction(); } - if (code.SUCESS == trx.getRet(0).getRet()) { + String code = trxExt.getResult().getCode().toString(); + if ("SUCCESS".equals(code)) { List list = trxExt.getConstantResultList(); byte[] listBytes = new byte[0]; for (ByteString bs : list) { From eaf2c34cbf52bf360b70a776d381a05d8a718f48 Mon Sep 17 00:00:00 2001 From: halibobo1205 <82020050+halibobo1205@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:20:12 +0800 Subject: [PATCH 57/61] feat(toolkit): show db root error to console (#5748) --- plugins/README.md | 11 ++++++ .../main/java/org/tron/plugins/DbRoot.java | 38 +++++++++++++++---- .../java/org/tron/plugins/DbRootTest.java | 2 +- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/plugins/README.md b/plugins/README.md index 19c952372c1..0db6f2e6143 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -134,3 +134,14 @@ Execute move command. java -jar Toolkit.jar db mv -c main_net_config.conf -d /data/tron/output-directory ``` +## DB Root + +DB root provides a helper which can compute merkle root for tiny db. + +NOTE: large db may GC overhead limit exceeded. + +### Available parameters: + +- ``: Source path for database. Default: output-directory/database +- `--db`: db name. +- `-h | --help`: provide the help info diff --git a/plugins/src/main/java/org/tron/plugins/DbRoot.java b/plugins/src/main/java/org/tron/plugins/DbRoot.java index 69555c16d33..7c33219e180 100644 --- a/plugins/src/main/java/org/tron/plugins/DbRoot.java +++ b/plugins/src/main/java/org/tron/plugins/DbRoot.java @@ -67,16 +67,20 @@ public Integer call() throws Exception { .errorText("Specify at least one exit database: --db dbName.")); return 404; } - if (dbs.size() > 1) { - ProgressBar.wrap(dbs.stream(), "root task").parallel().forEach(this::calcMerkleRoot); - } else { - calcMerkleRoot(dbs.get(0)); + List task = ProgressBar.wrap(dbs.stream(), "root task").parallel() + .map(this::calcMerkleRoot).collect(Collectors.toList()); + task.forEach(this::printInfo); + int code = (int) task.stream().filter(r -> r.code == 1).count(); + if (code > 0) { + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText("There are some errors, please check toolkit.log for detail.")); } spec.commandLine().getOut().println("root task done."); - return 0; + return code; } - private void calcMerkleRoot(String name) { + private Ret calcMerkleRoot(String name) { + Ret info = new Ret(); try (DBInterface database = DbTool.getDB(this.db, name)) { DBIterator iterator = database.iterator(); iterator.seekToFirst(); @@ -85,15 +89,33 @@ private void calcMerkleRoot(String name) { .collect(Collectors.toCollection(ArrayList::new)); Sha256Hash root = MerkleRoot.root(ids); logger.info("db: {},root: {}", database.getName(), root); - spec.commandLine().getOut().println(String.format("db: %s,root: %s", - database.getName(), root)); + info.code = 0; + info.msg = String.format("db: %s,root: %s", database.getName(), root); } catch (RocksDBException | IOException e) { logger.error("calc db {} fail", name, e); + info.code = 1; + info.msg = String.format("db: %s,fail: %s", + name, e.getMessage()); } + return info; } private Sha256Hash getHash(Map.Entry entry) { return Sha256Hash.of(true, Bytes.concat(entry.getKey(), entry.getValue())); } + + private void printInfo(Ret ret) { + if (ret.code == 0) { + spec.commandLine().getOut().println(ret.msg); + } else { + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText(ret.msg)); + } + } + + private static class Ret { + private int code; + private String msg; + } } diff --git a/plugins/src/test/java/org/tron/plugins/DbRootTest.java b/plugins/src/test/java/org/tron/plugins/DbRootTest.java index 4b2c6c7d373..b86688f77d5 100644 --- a/plugins/src/test/java/org/tron/plugins/DbRootTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbRootTest.java @@ -59,7 +59,7 @@ public void testRoot() throws IOException, RocksDBException { errorDb.put(("" + i).getBytes(), (ERROR_DB + "-" + i).getBytes()); } args = new String[] {"db", "root", database.toString(), "--db", ERROR_DB}; - Assert.assertEquals(0, cli.execute(args)); + Assert.assertEquals(1, cli.execute(args)); } } From 74c42e370688947c8be2fbe170f979fb13fa6d9b Mon Sep 17 00:00:00 2001 From: Brown Jiang <317787106@qq.com> Date: Mon, 11 Mar 2024 10:38:11 +0800 Subject: [PATCH 58/61] feat(conf):delete all bind.ip (#5750) --- framework/src/main/resources/config-backup.conf | 1 - framework/src/main/resources/config-beta.conf | 1 - framework/src/main/resources/config-localtest.conf | 1 - framework/src/main/resources/config-test-net.conf | 1 - framework/src/main/resources/config.conf | 2 +- framework/src/test/resources/config-localtest.conf | 1 - framework/src/test/resources/config-test-dbbackup.conf | 1 - framework/src/test/resources/config-test-index.conf | 1 - 8 files changed, 1 insertion(+), 8 deletions(-) diff --git a/framework/src/main/resources/config-backup.conf b/framework/src/main/resources/config-backup.conf index 2a1579fdc18..bb3082e42c2 100644 --- a/framework/src/main/resources/config-backup.conf +++ b/framework/src/main/resources/config-backup.conf @@ -51,7 +51,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "" external.ip = null } diff --git a/framework/src/main/resources/config-beta.conf b/framework/src/main/resources/config-beta.conf index 1d4a914ce20..050df1e45ad 100644 --- a/framework/src/main/resources/config-beta.conf +++ b/framework/src/main/resources/config-beta.conf @@ -51,7 +51,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "" external.ip = null } diff --git a/framework/src/main/resources/config-localtest.conf b/framework/src/main/resources/config-localtest.conf index 5ce30aeb1a4..f1ac104c9ed 100644 --- a/framework/src/main/resources/config-localtest.conf +++ b/framework/src/main/resources/config-localtest.conf @@ -57,7 +57,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "" external.ip = null } diff --git a/framework/src/main/resources/config-test-net.conf b/framework/src/main/resources/config-test-net.conf index 58c40587fcc..ff292a3951c 100644 --- a/framework/src/main/resources/config-test-net.conf +++ b/framework/src/main/resources/config-test-net.conf @@ -53,7 +53,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "" external.ip = null } diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index 5fe32b82880..78427c30f87 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -220,7 +220,7 @@ node { dns { # dns urls to get nodes, url format tree://{pubkey}@{domain}, default empty treeUrls = [ - #"tree://AKMQMNAJJBL73LXWPXDI4I5ZWWIZ4AWO34DWQ636QOBBXNFXH3LQS@main.trondisco.net", //offical dns tree + #"tree://AKMQMNAJJBL73LXWPXDI4I5ZWWIZ4AWO34DWQ636QOBBXNFXH3LQS@main.trondisco.net", ] } diff --git a/framework/src/test/resources/config-localtest.conf b/framework/src/test/resources/config-localtest.conf index fc35ca9b66a..d7f573fe90e 100644 --- a/framework/src/test/resources/config-localtest.conf +++ b/framework/src/test/resources/config-localtest.conf @@ -57,7 +57,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "" external.ip = null } diff --git a/framework/src/test/resources/config-test-dbbackup.conf b/framework/src/test/resources/config-test-dbbackup.conf index bb64a87ef69..4f9ddf8d32b 100644 --- a/framework/src/test/resources/config-test-dbbackup.conf +++ b/framework/src/test/resources/config-test-dbbackup.conf @@ -60,7 +60,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "" external.ip = null } diff --git a/framework/src/test/resources/config-test-index.conf b/framework/src/test/resources/config-test-index.conf index b9fbf5b2c43..6eb3621e747 100644 --- a/framework/src/test/resources/config-test-index.conf +++ b/framework/src/test/resources/config-test-index.conf @@ -54,7 +54,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "" external.ip = null } From 45a2af0f019d48ccbbe1f7ade36254713dfcc228 Mon Sep 17 00:00:00 2001 From: ss3344520 Date: Mon, 11 Mar 2024 14:11:47 +0800 Subject: [PATCH 59/61] feat(param): modify the default value of maxUnsolidifiedBlocks (#5751) * feat(param): modify the default value of maxUnsolidifiedBlocks --- .../src/main/java/org/tron/core/config/args/Args.java | 10 +++++----- .../test/java/org/tron/core/config/args/ArgsTest.java | 4 ++-- .../java/org/tron/core/net/TronNetDelegateTest.java | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 610e491c100..a8547b73948 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -229,8 +229,8 @@ public static void clearParam() { PARAMETER.dynamicConfigEnable = false; PARAMETER.dynamicConfigCheckInterval = 600; PARAMETER.allowTvmShangHai = 0; - PARAMETER.unsolidifiedBlockCheck = true; - PARAMETER.maxUnsolidifiedBlocks = 1000; + PARAMETER.unsolidifiedBlockCheck = false; + PARAMETER.maxUnsolidifiedBlocks = 54; PARAMETER.allowOldRewardOpt = 0; } @@ -1186,12 +1186,12 @@ public static void setParam(final String[] args, final String confFileName) { .getInt(Constant.COMMITTEE_ALLOW_TVM_SHANGHAI) : 0; PARAMETER.unsolidifiedBlockCheck = - !config.hasPath(Constant.UNSOLIDIFIED_BLOCK_CHECK) - || config.getBoolean(Constant.UNSOLIDIFIED_BLOCK_CHECK); + config.hasPath(Constant.UNSOLIDIFIED_BLOCK_CHECK) + && config.getBoolean(Constant.UNSOLIDIFIED_BLOCK_CHECK); PARAMETER.maxUnsolidifiedBlocks = config.hasPath(Constant.MAX_UNSOLIDIFIED_BLOCKS) ? config - .getInt(Constant.MAX_UNSOLIDIFIED_BLOCKS) : 1000; + .getInt(Constant.MAX_UNSOLIDIFIED_BLOCKS) : 54; long allowOldRewardOpt = config.hasPath(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) ? config .getInt(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) : 0; diff --git a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java index bd3d34e77f7..52307d9d294 100644 --- a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java +++ b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java @@ -99,8 +99,8 @@ public void get() { Assert.assertEquals(0, parameter.getActiveNodes().size()); Assert.assertEquals(30, parameter.getMaxConnections()); Assert.assertEquals(43, parameter.getNodeP2pVersion()); - Assert.assertEquals(1000, parameter.getMaxUnsolidifiedBlocks()); - Assert.assertEquals(true, parameter.isUnsolidifiedBlockCheck()); + Assert.assertEquals(54, parameter.getMaxUnsolidifiedBlocks()); + Assert.assertEquals(false, parameter.isUnsolidifiedBlockCheck()); //Assert.assertEquals(30, args.getSyncNodeCount()); // gRPC network configs checking diff --git a/framework/src/test/java/org/tron/core/net/TronNetDelegateTest.java b/framework/src/test/java/org/tron/core/net/TronNetDelegateTest.java index c1ac3df1599..727aad9dccb 100644 --- a/framework/src/test/java/org/tron/core/net/TronNetDelegateTest.java +++ b/framework/src/test/java/org/tron/core/net/TronNetDelegateTest.java @@ -20,6 +20,7 @@ public void test() throws Exception { Args.setParam(new String[] {"-w"}, Constant.TEST_CONF); CommonParameter parameter = Args.getInstance(); Args.logConfig(); + parameter.setUnsolidifiedBlockCheck(true); BlockCapsule.BlockId blockId = new BlockCapsule.BlockId(Sha256Hash.ZERO_HASH, 10000L); From 3b5ed16cdae99a1b33ca6b6f80ac93b0fc14afc2 Mon Sep 17 00:00:00 2001 From: fyyhtx <72650068+fyyhtx@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:33:56 +0800 Subject: [PATCH 60/61] feat(net): optimize block processing logic (#5754) feat(net): optimize block processing logic --- .../net/messagehandler/BlockMsgHandler.java | 16 +++++++++------- .../messagehandler/PbftDataSyncHandler.java | 18 +++++++++++++----- .../net/messagehandler/PbftMsgHandler.java | 3 +++ .../PbftDataSyncHandlerTest.java | 7 ++++--- .../net/messagehandler/PbftMsgHandlerTest.java | 9 +++++++++ 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java index 9bb746346a6..14523df86a5 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java @@ -125,6 +125,14 @@ private void check(PeerConnection peer, BlockMessage msg) throws P2pException { private void processBlock(PeerConnection peer, BlockCapsule block) throws P2pException { BlockId blockId = block.getBlockId(); + boolean flag = tronNetDelegate.validBlock(block); + if (!flag) { + logger.warn("Receive a bad block from {}, {}, {}", + peer.getInetSocketAddress(), blockId.getString(), + Hex.toHexString(block.getWitnessAddress().toByteArray())); + return; + } + if (!tronNetDelegate.containBlock(block.getParentBlockId())) { logger.warn("Get unlink block {} from {}, head is {}", blockId.getString(), peer.getInetAddress(), tronNetDelegate.getHeadBlockId().getString()); @@ -138,16 +146,10 @@ private void processBlock(PeerConnection peer, BlockCapsule block) throws P2pExc return; } - boolean flag = tronNetDelegate.validBlock(block); - if (flag) { - broadcast(new BlockMessage(block)); - } + broadcast(new BlockMessage(block)); try { tronNetDelegate.processBlock(block, false); - if (!flag) { - broadcast(new BlockMessage(block)); - } witnessProductBlockService.validWitnessProductTwoBlock(block); diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java index 60f614632a4..d66fa6d41f7 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java @@ -1,5 +1,7 @@ package org.tron.core.net.messagehandler; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.Sets; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; @@ -8,12 +10,11 @@ import java.security.SignatureException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -37,7 +38,9 @@ @Service public class PbftDataSyncHandler implements TronMsgHandler, Closeable { - private Map pbftCommitMessageCache = new ConcurrentHashMap<>(); + private static final Cache pbftCommitMessageCache = + CacheBuilder.newBuilder().initialCapacity(100).maximumSize(200) + .expireAfterWrite(10, TimeUnit.MINUTES).build(); private final String esName = "valid-header-pbft-sign"; @@ -51,6 +54,9 @@ public class PbftDataSyncHandler implements TronMsgHandler, Closeable { public void processMessage(PeerConnection peer, TronMessage msg) throws P2pException { PbftCommitMessage pbftCommitMessage = (PbftCommitMessage) msg; try { + if (!chainBaseManager.getDynamicPropertiesStore().allowPBFT()) { + return; + } Raw raw = Raw.parseFrom(pbftCommitMessage.getPBFTCommitResult().getData()); pbftCommitMessageCache.put(raw.getViewN(), pbftCommitMessage); } catch (InvalidProtocolBufferException e) { @@ -64,7 +70,8 @@ public void processPBFTCommitData(BlockCapsule block) { return; } long epoch = 0; - PbftCommitMessage pbftCommitMessage = pbftCommitMessageCache.remove(block.getNum()); + PbftCommitMessage pbftCommitMessage = pbftCommitMessageCache.getIfPresent(block.getNum()); + pbftCommitMessageCache.invalidate(block.getNum()); long maintenanceTimeInterval = chainBaseManager.getDynamicPropertiesStore() .getMaintenanceTimeInterval(); if (pbftCommitMessage == null) { @@ -75,7 +82,8 @@ public void processPBFTCommitData(BlockCapsule block) { Raw raw = Raw.parseFrom(pbftCommitMessage.getPBFTCommitResult().getData()); epoch = raw.getEpoch(); } - pbftCommitMessage = pbftCommitMessageCache.remove(epoch); + pbftCommitMessage = pbftCommitMessageCache.getIfPresent(epoch); + pbftCommitMessageCache.invalidate(epoch); if (pbftCommitMessage != null) { processPBFTCommitMessage(pbftCommitMessage); } diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/PbftMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/PbftMsgHandler.java index 4e4cc858898..d086cc28b6c 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/PbftMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/PbftMsgHandler.java @@ -33,6 +33,9 @@ public class PbftMsgHandler { private TronNetDelegate tronNetDelegate; public void processMessage(PeerConnection peer, PbftMessage msg) throws Exception { + if (!tronNetDelegate.allowPBFT()) { + return; + } if (Param.getInstance().getPbftInterface().isSyncing()) { return; } diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/PbftDataSyncHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/PbftDataSyncHandlerTest.java index d1fdfaa5d90..e5d242a6c4d 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/PbftDataSyncHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/PbftDataSyncHandlerTest.java @@ -32,9 +32,6 @@ public void testProcessMessage() throws Exception { Protocol.PBFTMessage.Raw raw = rawBuilder.build(); PbftSignCapsule pbftSignCapsule = new PbftSignCapsule(raw.toByteString(), new ArrayList<>()); PbftCommitMessage pbftCommitMessage = new PbftCommitMessage(pbftSignCapsule); - pbftDataSyncHandler.processMessage(null, pbftCommitMessage); - Assert.assertEquals(Protocol.PBFTMessage.Raw.parseFrom( - pbftCommitMessage.getPBFTCommitResult().getData()).getViewN(), 1); DynamicPropertiesStore dynamicPropertiesStore = Mockito.mock(DynamicPropertiesStore.class); PbftSignDataStore pbftSignDataStore = Mockito.mock(PbftSignDataStore.class); @@ -48,6 +45,10 @@ public void testProcessMessage() throws Exception { field.setAccessible(true); field.set(pbftDataSyncHandler, chainBaseManager); + pbftDataSyncHandler.processMessage(null, pbftCommitMessage); + Assert.assertEquals(Protocol.PBFTMessage.Raw.parseFrom( + pbftCommitMessage.getPBFTCommitResult().getData()).getViewN(), 1); + pbftDataSyncHandler.processPBFTCommitData(blockCapsule); Field field1 = PbftDataSyncHandler.class.getDeclaredField("pbftCommitMessageCache"); field1.setAccessible(true); diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/PbftMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/PbftMsgHandlerTest.java index 3fec10fc163..8b9d1969cfc 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/PbftMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/PbftMsgHandlerTest.java @@ -34,6 +34,7 @@ import org.tron.core.net.message.MessageTypes; import org.tron.core.net.peer.PeerConnection; import org.tron.core.net.peer.PeerManager; +import org.tron.core.store.DynamicPropertiesStore; import org.tron.p2p.P2pConfig; import org.tron.p2p.base.Parameter; import org.tron.p2p.connection.Channel; @@ -116,6 +117,14 @@ public void testPbft() throws Exception { Assert.assertEquals(P2pException.TypeEnum.BAD_MESSAGE, e.getType()); } + DynamicPropertiesStore dynamicPropertiesStore = context.getBean(DynamicPropertiesStore.class); + dynamicPropertiesStore.saveAllowPBFT(1); + try { + context.getBean(PbftMsgHandler.class).processMessage(peer, pbftMessage); + } catch (P2pException e) { + Assert.assertEquals(P2pException.TypeEnum.BAD_MESSAGE, e.getType()); + } + Assert.assertEquals(1, PeerManager.getPeers().size()); } } From 9532bc028f228b3fb66a7b0cca56db47cefaad0c Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Fri, 15 Mar 2024 11:50:54 +0800 Subject: [PATCH 61/61] remove unused test --- .../org/tron/common/cache/CacheManagerTest.java | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 framework/src/main/java/org/tron/common/cache/CacheManagerTest.java diff --git a/framework/src/main/java/org/tron/common/cache/CacheManagerTest.java b/framework/src/main/java/org/tron/common/cache/CacheManagerTest.java deleted file mode 100644 index b02b135552a..00000000000 --- a/framework/src/main/java/org/tron/common/cache/CacheManagerTest.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.tron.common.cache; - -import org.junit.Assert; -import org.junit.Test; - -public class CacheManagerTest { - - @Test - public void allocate() { - String strategy = String.format(CacheStrategies.PATTERNS, 1, 1, "30s", 1); - TronCache cache = CacheManager.allocate(CacheType.witnessStandby, strategy); - Assert.assertNull(cache.getIfPresent("test")); - } -}