From 7a21b86018e48a9ce967bd2053de4a2f5e8a9dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Thu, 22 Aug 2024 09:48:26 +0300 Subject: [PATCH 01/15] chore: current changes --- .../com/limechain/client/LightClient.java | 6 +- .../java/com/limechain/network/Network.java | 26 ++--- .../network/StrictProtocolBinding.java | 58 +++++++++--- .../network/kad/KademliaService.java | 70 ++++---------- .../protocol/blockannounce/BlockAnnounce.java | 2 +- .../network/protocol/grandpa/Grandpa.java | 2 +- .../network/protocol/warp/WarpSync.java | 31 ++++-- .../protocol/warp/WarpSyncController.java | 8 +- .../protocol/warp/WarpSyncProtocol.java | 94 ++++++++++++++----- .../warp/scale/reader/BlockHeaderReader.java | 6 ++ .../com/limechain/network/wrapper/Stream.java | 29 ++++++ .../polkaj/reader/ScaleCodecReader.java | 2 +- .../sync/warpsync/WarpSyncMachine.java | 7 +- .../action/RequestFragmentsAction.java | 48 +++++----- .../java/com/limechain/utils/StringUtils.java | 35 +++---- src/main/webapp/index.html | 8 +- 16 files changed, 269 insertions(+), 163 deletions(-) create mode 100644 src/main/java/com/limechain/network/wrapper/Stream.java diff --git a/src/main/java/com/limechain/client/LightClient.java b/src/main/java/com/limechain/client/LightClient.java index 0e5bcc969..bdcf1e934 100644 --- a/src/main/java/com/limechain/client/LightClient.java +++ b/src/main/java/com/limechain/client/LightClient.java @@ -33,15 +33,15 @@ public LightClient() { @SneakyThrows public void start() { this.network.start(); - + System.out.println("Network started"); while (true) { + this.network.updateCurrentSelectedPeer(); if (this.network.getKademliaService().getSuccessfulBootNodes() > 0) { log.log(Level.INFO, "Node successfully connected to a peer! Sync can start!"); this.warpSyncMachine = AppBean.getBean(WarpSyncMachine.class); + System.out.println("Starting warp sync machine..."); this.warpSyncMachine.start(); break; - } else { - this.network.updateCurrentSelectedPeer(); } log.log(Level.INFO, "Waiting for peer connection..."); Thread.sleep(10000); diff --git a/src/main/java/com/limechain/network/Network.java b/src/main/java/com/limechain/network/Network.java index 8abe59c6a..2c78cf58d 100644 --- a/src/main/java/com/limechain/network/Network.java +++ b/src/main/java/com/limechain/network/Network.java @@ -4,6 +4,8 @@ import com.limechain.chain.ChainService; import com.limechain.config.HostConfig; import com.limechain.network.kad.KademliaService; +import com.limechain.network.protocol.warp.WarpSyncService; +import com.limechain.network.protocol.warp.dto.WarpSyncResponse; import com.limechain.rpc.server.AppBean; import com.limechain.sync.warpsync.WarpSyncState; import lombok.Getter; @@ -50,15 +52,18 @@ private void initializeProtocols(ChainService chainService, HostConfig hostConfig) { // -// String chainId = chainService.getChainSpec().getProtocolId(); -// String warpProtocolId = ProtocolUtils.getWarpSyncProtocol(chainId); + String chainId = chainService.getChainSpec().getProtocolId(); + String warpProtocolId = ProtocolUtils.getWarpSyncProtocol(chainId); // String lightProtocolId = ProtocolUtils.getLightMessageProtocol(chainId); // String blockAnnounceProtocolId = ProtocolUtils.getBlockAnnounceProtocol(chainId); // String grandpaProtocolId = ProtocolUtils.getGrandpaProtocol(chainId); kademliaService = new KademliaService(); + warpSyncService = new WarpSyncService(warpProtocolId); } + WarpSyncService warpSyncService; + // private Ed25519PrivateKey loadPrivateKeyFromDB(KVRepository repository) { // Ed25519PrivateKey privateKey; // @@ -110,6 +115,7 @@ public void updateCurrentSelectedPeer() { // if (connectionManager.getPeerIds().isEmpty()) return; // this.currentSelectedPeer = connectionManager.getPeerIds().stream() // .skip(RANDOM.nextInt(connectionManager.getPeerIds().size())).findAny().orElse(null); + kademliaService.updateSuccessfulBootNodes(); } // public String getPeerId() { @@ -138,7 +144,7 @@ public void findPeers() { } if (getPeersCount() >= REPLICATION) { log.log(Level.INFO, - "Connections have reached replication factor(" + REPLICATION + "). " + + "Connections have reached replication factor(" + REPLICATION + "). " + "No need to search for new ones yet."); return; } @@ -177,7 +183,7 @@ public void pingPeers() { // } // } -// public BlockResponse syncBlock(PeerId peerId, BigInteger lastBlockNumber) { + // public BlockResponse syncBlock(PeerId peerId, BigInteger lastBlockNumber) { // this.currentSelectedPeer = peerId; // // TODO: fields, hash, direction and maxBlocks values not verified // // TODO: when debugging could not get a value returned @@ -202,14 +208,12 @@ public void pingPeers() { // ); // } // -// public WarpSyncResponse makeWarpSyncRequest(String blockHash) { + public WarpSyncResponse makeWarpSyncRequest(String blockHash) { // if (isPeerInvalid()) return null; -// -// return this.warpSyncService.getProtocol().warpSyncRequest( -// this.host, -// this.currentSelectedPeer, -// blockHash); -// } + + return this.warpSyncService.getProtocol().warpSyncRequest( + blockHash); + } // // public LightClientMessage.Response makeRemoteReadRequest(String blockHash, String[] keys) { // if (isPeerInvalid()) return null; diff --git a/src/main/java/com/limechain/network/StrictProtocolBinding.java b/src/main/java/com/limechain/network/StrictProtocolBinding.java index 6c7d8b075..a33898dc9 100644 --- a/src/main/java/com/limechain/network/StrictProtocolBinding.java +++ b/src/main/java/com/limechain/network/StrictProtocolBinding.java @@ -1,18 +1,52 @@ package com.limechain.network; -public abstract class StrictProtocolBinding { +import com.limechain.network.wrapper.Stream; +import org.teavm.jso.JSBody; +import org.teavm.jso.core.JSPromise; + +import java.util.concurrent.atomic.AtomicReference; + +public abstract class StrictProtocolBinding { + String protocolId; + protected StrictProtocolBinding(String protocolId/*, T protocol*/) { + this.protocolId = protocolId; + } + + public Stream dialPeer(/*PeerId peer*/) { + Object peer1 = getPeer(); + System.out.println("Peer: " + peer1); + JSPromise dial = dial(peer1, protocolId); + final var lock = new Object(); + AtomicReference stream = new AtomicReference<>(); + + dial.then((result) -> { + stream.set((Stream) result); + synchronized (lock) { + lock.notify(); + } + return null; + }); + + synchronized (lock) { + try { + lock.wait(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + Stream atomicStream = stream.get(); + + System.out.println("Stream: " + toJson(atomicStream)); + return atomicStream; } -// public T dialPeer(Host us, PeerId peer, AddressBook addrs) { -// Multiaddr[] addr = addrs.get(peer) -// .join().stream() -// .filter(address -> !address.toString().contains("/ws") && !address.toString().contains("/wss")) -// .toList() -// .toArray(new Multiaddr[0]); -// if (addr.length == 0) -// throw new IllegalStateException("No addresses known for peer " + peer); -// -// return dial(us, peer, addr).getController().join(); -// } + @JSBody(params = {"object"}, script = "return JSON.stringify(object);") + private static native String toJson(Object object); + + @JSBody(params = {"peerId", "protocolId"}, script = "return (async () => ItPbStream.pbStream(await libp.dialProtocol(peerId, protocolId)))()") + private static native JSPromise dial(Object peerId, String protocolId); + + @JSBody(script = "return libp.getConnections()[0].remotePeer;") + private static native Object getPeer(); } diff --git a/src/main/java/com/limechain/network/kad/KademliaService.java b/src/main/java/com/limechain/network/kad/KademliaService.java index 9ff049188..3e424564b 100644 --- a/src/main/java/com/limechain/network/kad/KademliaService.java +++ b/src/main/java/com/limechain/network/kad/KademliaService.java @@ -2,6 +2,7 @@ import com.limechain.network.kad.dto.Host; import com.limechain.network.kad.dto.PeerId; +import com.limechain.network.protocol.NetworkService; import lombok.Getter; import lombok.Setter; import lombok.extern.java.Log; @@ -9,6 +10,7 @@ import org.teavm.interop.AsyncCallback; import org.teavm.jso.JSBody; +import java.util.ArrayList; import java.util.Arrays; import java.util.Random; import java.util.concurrent.ExecutionException; @@ -19,31 +21,13 @@ */ @Getter @Log -public class KademliaService /*extends NetworkService*/ { +public class KademliaService extends NetworkService { public static final int REPLICATION = 20; - private static final Random RANDOM = new Random(); @Setter private Host host; - // private List bootNodePeerIds; private int successfulBootNodes; - - public void addReservedPeer(String multiaddr) throws ExecutionException, InterruptedException { -// final Multiaddr addrWithPeer = Multiaddr.fromString(multiaddr); -// -// CompletableFuture peerStream = -// protocol.dial(host, addrWithPeer.getPeerId(), addrWithPeer).getStream(); -// -// Stream stream = peerStream.get(); -// if (stream == null) { -// log.log(Level.WARNING, "Failed to connect to reserved peer"); -// } else { -// ConnectionManager.getInstance().addNewPeer(addrWithPeer.getPeerId()); -// log.log(Level.INFO, "Successfully connected to reserved peer"); -// } - } - /** * Connects to boot nodes to the Kademlia dht * @@ -76,49 +60,35 @@ public int connectBootNodes(String[] bootNodes) { return successfulBootNodes; } + public void updateSuccessfulBootNodes() { + successfulBootNodes = getPeerStoreSize(); + } + @JSBody(params = {"bootNodes"}, script = "start(bootNodes)") - @Async public static native void startNetwork(String[] bootNodes); @JSBody(script = "return getPeerId()") public static native Object getPeerId(); + @JSBody(script = "return libp.peerId.privateKey") public static native byte[] getPeerPrivateKey(); + @JSBody(script = "return libp.peerId.publicKey") public static native byte[] getPeerPublicKey(); - @JSBody(script = "return libp.peerStore.store.datastore.data.size") - public static native int getPeerStoreSize(); + @JSBody(script = "return libp.getConnections().length") + public static native int getPeerStoreSize(); /** * Populates Kademlia dht with peers closest in distance to a random id then makes connections with our node */ - public void findNewPeers() { -// protocol.findClosestPeers(randomPeerId(), REPLICATION, host); -// final var peers = -// protocol.findClosestPeers(Multihash.deserialize(host.getPeerId().getBytes()), REPLICATION, host); -// -// peers.stream().parallel().forEach(p -> { -// boolean isConnected = protocol.connectTo(host, p); -// -// if (!isConnected) { -// protocol.connectTo(host, p); -// } -// }); - } -// -// private Multihash randomPeerId() { -// byte[] hash = new byte[32]; -// RANDOM.nextBytes(hash); -// return new Multihash(Multihash.Type.sha2_256, hash); -// } -// -// private void setBootNodePeerIds(String[] bootNodes) { -// ArrayList ids = new ArrayList<>(); -// for (String bootNode : bootNodes) { -// String peerId = bootNode.substring(bootNode.lastIndexOf('/') + 1, bootNode.length()); -// ids.add(PeerId.fromBase58(peerId)); -// } -// this.bootNodePeerIds = ids; -// } + @JSBody(script = "libp.peerStore.forEach( async (p) => {" + + " for await (const foundPeer of dht.peerRouting.getClosestPeers(p.id.toBytes())){" + + " if(foundPeer.peer?.multiaddrs?.length > 0){" + + " try{libp.dial(foundPeer.peer)}finally{}" + + " }" + + " }" + + "});") + public static native void findNewPeers(); + } \ No newline at end of file diff --git a/src/main/java/com/limechain/network/protocol/blockannounce/BlockAnnounce.java b/src/main/java/com/limechain/network/protocol/blockannounce/BlockAnnounce.java index 7d5fd5a95..a9f26513c 100644 --- a/src/main/java/com/limechain/network/protocol/blockannounce/BlockAnnounce.java +++ b/src/main/java/com/limechain/network/protocol/blockannounce/BlockAnnounce.java @@ -4,7 +4,7 @@ import lombok.extern.java.Log; @Log -public class BlockAnnounce extends StrictProtocolBinding { +public class BlockAnnounce extends StrictProtocolBinding { public BlockAnnounce(String protocolId, BlockAnnounceProtocol protocol) { super(protocolId/*, protocol*/); } diff --git a/src/main/java/com/limechain/network/protocol/grandpa/Grandpa.java b/src/main/java/com/limechain/network/protocol/grandpa/Grandpa.java index 24e62eeb5..1bc1fee4d 100644 --- a/src/main/java/com/limechain/network/protocol/grandpa/Grandpa.java +++ b/src/main/java/com/limechain/network/protocol/grandpa/Grandpa.java @@ -5,7 +5,7 @@ /** * GRANDPA protocol binding */ -public class Grandpa extends StrictProtocolBinding { +public class Grandpa extends StrictProtocolBinding { public Grandpa(String protocolId, GrandpaProtocol protocol) { super(protocolId/*, protocol*/); } diff --git a/src/main/java/com/limechain/network/protocol/warp/WarpSync.java b/src/main/java/com/limechain/network/protocol/warp/WarpSync.java index bf0e07b03..5e6075a56 100644 --- a/src/main/java/com/limechain/network/protocol/warp/WarpSync.java +++ b/src/main/java/com/limechain/network/protocol/warp/WarpSync.java @@ -1,21 +1,38 @@ package com.limechain.network.protocol.warp; +import com.limechain.exception.global.ExecutionFailedException; +import com.limechain.exception.global.ThreadInterruptedException; import com.limechain.network.StrictProtocolBinding; +import com.limechain.network.kad.dto.Host; +import com.limechain.network.kad.dto.PeerId; +import com.limechain.network.protocol.warp.dto.WarpSyncRequest; +import com.limechain.network.protocol.warp.dto.WarpSyncResponse; +import com.limechain.network.wrapper.Stream; import lombok.extern.java.Log; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; + @Log -public class WarpSync extends StrictProtocolBinding { +public class WarpSync extends StrictProtocolBinding { + + private String protocolId; public WarpSync(String protocolId, WarpSyncProtocol protocol) { super(protocolId/*, protocol*/); + this.protocolId = protocolId; } -// public WarpSyncResponse warpSyncRequest(Host us, PeerId peer, String blockHash) { + public WarpSyncResponse warpSyncRequest(/*PeerId peer,*/ String blockHash) { // try { -// WarpSyncController controller = dialPeer(us, peer, us.getAddressBook()); -// WarpSyncResponse resp = controller.warpSyncRequest(blockHash).get(10, TimeUnit.SECONDS); -// log.log(Level.INFO, "Received warp sync response with " + resp.getFragments().length + " fragments"); -// return resp; +// Stream stream = dialPeer(/*peer*/); + WarpSyncProtocol.Sender sender = new WarpSyncProtocol.Sender(); + System.out.println("Block hash: " + blockHash); + WarpSyncResponse resp = sender.warpSyncRequest(blockHash, protocolId); + log.log(Level.INFO, "Received warp sync response with " + resp/*.getFragments().length*/ + " fragments"); + return resp; // } catch (ExecutionException | TimeoutException | IllegalStateException e) { // log.log(Level.SEVERE, "Error while sending remote call request: ", e); // throw new ExecutionFailedException(e); @@ -23,5 +40,5 @@ public WarpSync(String protocolId, WarpSyncProtocol protocol) { // Thread.currentThread().interrupt(); // throw new ThreadInterruptedException(e); // } -// } + } } diff --git a/src/main/java/com/limechain/network/protocol/warp/WarpSyncController.java b/src/main/java/com/limechain/network/protocol/warp/WarpSyncController.java index 9a2c3c234..f8fb14d42 100644 --- a/src/main/java/com/limechain/network/protocol/warp/WarpSyncController.java +++ b/src/main/java/com/limechain/network/protocol/warp/WarpSyncController.java @@ -3,14 +3,12 @@ import com.limechain.network.protocol.warp.dto.WarpSyncRequest; import com.limechain.network.protocol.warp.dto.WarpSyncResponse; -import java.util.concurrent.CompletableFuture; - public interface WarpSyncController { - CompletableFuture send(WarpSyncRequest req); + WarpSyncResponse send(WarpSyncRequest req, String protocolId); - default CompletableFuture warpSyncRequest(String blockHash) { + default WarpSyncResponse warpSyncRequest(String blockHash, String protocolId) { var request = new WarpSyncRequest(blockHash); - return send(request); + return send(request, protocolId); } } diff --git a/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java b/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java index ec288eb6a..dc704f88b 100644 --- a/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java +++ b/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java @@ -1,5 +1,19 @@ package com.limechain.network.protocol.warp; +import com.limechain.network.protocol.warp.dto.WarpSyncRequest; +import com.limechain.network.protocol.warp.dto.WarpSyncResponse; +import com.limechain.network.protocol.warp.scale.reader.WarpSyncResponseScaleReader; +import com.limechain.network.wrapper.Stream; +import com.limechain.polkaj.reader.ScaleCodecReader; +import com.limechain.utils.StringUtils; +import org.teavm.jso.JSBody; +import org.teavm.jso.core.JSArray; +import org.teavm.jso.core.JSPromise; + +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; + public class WarpSyncProtocol /*extends ProtocolHandler*/ { // Sizes taken from smoldot public static final int MAX_REQUEST_SIZE = 32; @@ -21,39 +35,71 @@ protected CompletableFuture onStartInitiator(Stream stream) return CompletableFuture.completedFuture(handler); }*/ - /*static class Sender implements ProtocolMessageHandler, WarpSyncController { + static class Sender implements WarpSyncController { public static final int MAX_QUEUE_SIZE = 50; - private final LinkedBlockingDeque> queue = - new LinkedBlockingDeque<>(MAX_QUEUE_SIZE); - private final Stream stream; +// private final LinkedBlockingDeque> queue = +// new LinkedBlockingDeque<>(MAX_QUEUE_SIZE); - public Sender(Stream stream) { - this.stream = stream; + public Sender() { } - @Override + // @Override public void onMessage(Stream stream, WarpSyncResponse msg) { - Objects.requireNonNull(queue.poll()).complete(msg); - stream.closeWrite(); +// Objects.requireNonNull(queue.poll()).complete(msg); +// stream.closeWrite(); } @Override - public CompletableFuture send(WarpSyncRequest req) { - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - try (ScaleCodecWriter writer = new ScaleCodecWriter(buf)) { - writer.write(new WarpSyncRequestWriter(), req); - } catch (IOException e) { - throw new ScaleEncodingException(e); + public WarpSyncResponse send(WarpSyncRequest req, String protocolId) { + System.out.println("Request: " + req.getBlockHash()); + final var lock = new Object(); + + JSPromise> objectJSPromise = sendRequest(StringUtils.toHex(req.getBlockHash().getBytes()), protocolId); + + objectJSPromise.then((ttt) -> { + System.out.println(ttt); + System.out.println(ttt.get(0)); + System.out.println(ttt.get(ttt.getLength() - 1)); + byte[] bytes = new byte[ttt.getLength()]; + for (int i = 0; i < ttt.getLength(); i++) { + //bytes[i] = (byte) ((int) ttt.get(i)); //fails here + } +// byte[] bytes = StringUtils.fromHex(ttt); + System.out.println("Received response: " + " " + bytes); + System.out.println("Received response len: " + bytes.length); + + synchronized (lock) { +// System.out.println("Received response: " + bytes.length + " " + bytes); + ScaleCodecReader scaleCodecReader = new ScaleCodecReader(bytes); + WarpSyncResponse responseaa = new WarpSyncResponseScaleReader().read(scaleCodecReader); + System.out.println(responseaa); +// response.set(result); + lock.notify(); + } + return null; + }); + + synchronized (lock) { + try { + lock.wait(); +// byte[] bytes = response.get(); +// System.out.println("Received response: " + /*bytes.length +*/ " " + bytes); +// ScaleCodecReader scaleCodecReader = new ScaleCodecReader(bytes); +// WarpSyncResponse responseaa = new WarpSyncResponseScaleReader().read(scaleCodecReader); +// System.out.println(responseaa); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } - CompletableFuture res = new CompletableFuture<>(); - queue.add(res); - stream.writeAndFlush(buf.toByteArray()); - return res; + return null; } - @Override - public void onException(Throwable cause) { - Objects.requireNonNull(queue.poll()).completeExceptionally(cause); - } - }*/ + @JSBody(params = {"blockHash", "protocolId"}, script = "return (async () => {" + + "let peer = libp.getConnections()[0].remotePeer;" + + "let stream = await ItPbStream.pbStream(await libp.dialProtocol(peer, protocolId));" + + "stream.writeLP(new Uint8Array([...blockHash.matchAll(/../g)].map(m => parseInt(m[0], 16))));" + + "return Array.from((await stream.readLP()).subarray());})()") + private static native JSPromise> sendRequest(String blockHash, String protocolId); + + } } diff --git a/src/main/java/com/limechain/network/protocol/warp/scale/reader/BlockHeaderReader.java b/src/main/java/com/limechain/network/protocol/warp/scale/reader/BlockHeaderReader.java index b40df4c4a..0c9974ea8 100644 --- a/src/main/java/com/limechain/network/protocol/warp/scale/reader/BlockHeaderReader.java +++ b/src/main/java/com/limechain/network/protocol/warp/scale/reader/BlockHeaderReader.java @@ -12,12 +12,17 @@ public class BlockHeaderReader implements ScaleReader { @Override public BlockHeader read(ScaleCodecReader reader) { BlockHeader blockHeader = new BlockHeader(); + System.out.println("BlockHeaderReader.read"); blockHeader.setParentHash(new Hash256(reader.readUint256())); + System.out.println("BlockHeaderReader.setParentHash"); // NOTE: Usage of BlockNumberReader is intentionally omitted here, // since we want this to be a compact int, not a var size int blockHeader.setBlockNumber(BigInteger.valueOf(reader.readCompactInt())); + System.out.println("BlockHeaderReader.setBlockNumber"); blockHeader.setStateRoot(new Hash256(reader.readUint256())); + System.out.println("BlockHeaderReader.setStateRoot"); blockHeader.setExtrinsicsRoot(new Hash256(reader.readUint256())); + System.out.println("BlockHeaderReader.setExtrinsicsRoot"); var digestCount = reader.readCompactInt(); HeaderDigest[] digests = new HeaderDigest[digestCount]; @@ -26,6 +31,7 @@ public BlockHeader read(ScaleCodecReader reader) { } blockHeader.setDigest(digests); + System.out.println("BlockHeaderReader.setDigest"); return blockHeader; } diff --git a/src/main/java/com/limechain/network/wrapper/Stream.java b/src/main/java/com/limechain/network/wrapper/Stream.java new file mode 100644 index 000000000..590049edd --- /dev/null +++ b/src/main/java/com/limechain/network/wrapper/Stream.java @@ -0,0 +1,29 @@ +package com.limechain.network.wrapper; + +import org.teavm.jso.JSObject; + +public class Stream implements JSObject { + public void write() { + + } + + public void writeLP(byte[] data) { + + } + + public void writePB() { + + } + + public void read() { + + } + + public Object readLP() { + return null; + } + + public void readPB() { + + } +} diff --git a/src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java b/src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java index 7ab6c8d3c..6348a2310 100644 --- a/src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java +++ b/src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java @@ -29,7 +29,7 @@ public ScaleCodecReader(byte[] source) { * @return true if has more elements */ public boolean hasNext() { - return pos < source.length; + return true;//pos < source.length; } /** diff --git a/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java b/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java index 2edeecf22..f7b0a6ed2 100644 --- a/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java +++ b/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java @@ -54,6 +54,7 @@ public void nextState() { } public void handleState() { + System.out.println("Warp sync action" + warpSyncAction.getClass().getSimpleName()); warpSyncAction.handle(this); } @@ -72,14 +73,14 @@ public void start() { final Hash256 initStateHash = this.syncState.getLastFinalizedBlockHash(); // Always start with requesting fragments - log.log(Level.INFO, "Requesting fragments..."); + log.log(Level.INFO, "Requesting fragments... " + initStateHash.toString()); this.networkService.updateCurrentSelectedPeerWithNextBootnode(); this.warpSyncAction = new RequestFragmentsAction(initStateHash); // new Thread(() -> { // while (this.warpSyncAction.getClass() != FinishedAction.class) { -// this.handleState(); -// this.nextState(); + this.handleState(); + this.nextState(); // } // // finishWarpSync(); diff --git a/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java b/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java index b5861ad2b..671f035dc 100644 --- a/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java +++ b/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java @@ -1,5 +1,7 @@ package com.limechain.sync.warpsync.action; +import com.limechain.exception.global.MissingObjectException; +import com.limechain.network.protocol.warp.dto.WarpSyncResponse; import com.limechain.polkaj.Hash256; import com.limechain.rpc.server.AppBean; import com.limechain.sync.warpsync.WarpSyncMachine; @@ -13,7 +15,7 @@ public class RequestFragmentsAction implements WarpSyncAction { private final WarpSyncState warpSyncState; private final Hash256 blockHash; -// private WarpSyncResponse result; + // private WarpSyncResponse result; private Exception error; public RequestFragmentsAction(Hash256 blockHash) { @@ -37,7 +39,7 @@ public void next(WarpSyncMachine sync) { } } // if (this.result != null) { - sync.setWarpSyncAction(new VerifyJustificationAction()); + sync.setWarpSyncAction(new VerifyJustificationAction()); // return; // } log.log(Level.WARNING, "RequestFragmentsState.next() called without result or error set."); @@ -45,10 +47,10 @@ public void next(WarpSyncMachine sync) { @Override public void handle(WarpSyncMachine sync) { -// WarpSyncResponse resp = null; + WarpSyncResponse resp = null; // for (int i = 0; i < sync.getNetworkService().getKademliaService().getBootNodePeerIds().size(); i++) { // try { -// resp = sync.getNetworkService().makeWarpSyncRequest(blockHash.toString()); + resp = sync.getNetworkService().makeWarpSyncRequest(blockHash.toString()); // break; // } catch (Exception e) { // if (!sync.getNetworkService().updateCurrentSelectedPeerWithBootnode(i)) { @@ -57,28 +59,28 @@ public void handle(WarpSyncMachine sync) { // } // } // } -// try { -// if (resp == null) { -// throw new MissingObjectException("No response received."); -// } -// -// log.log(Level.INFO, "Successfully received fragments from peer " -// + sync.getNetworkService().getCurrentSelectedPeer()); -// if (resp.getFragments().length == 0) { -// log.log(Level.WARNING, "No fragments received."); -// return; -// } -// warpSyncState.setWarpSyncFragmentsFinished(resp.isFinished()); + try { + if (resp == null) { + throw new MissingObjectException("No response received."); + } + + log.log(Level.INFO, "Successfully received fragments from peer " + /* + sync.getNetworkService().getCurrentSelectedPeer()*/); + if (resp.getFragments().length == 0) { + log.log(Level.WARNING, "No fragments received."); + return; + } + warpSyncState.setWarpSyncFragmentsFinished(resp.isFinished()); // sync.setFragmentsQueue(new LinkedBlockingQueue<>( // Arrays.stream(resp.getFragments()).toList()) // ); -// + // this.result = resp; -// } catch (Exception e) { -// // TODO: Set error state, next() will use to transition to correct next state. -// // This error state could be either recoverable or irrecoverable. -// log.log(Level.WARNING, "Error while requesting fragments: " + e.getMessage()); -// this.error = e; -// } + } catch (Exception e) { + // TODO: Set error state, next() will use to transition to correct next state. + // This error state could be either recoverable or irrecoverable. + log.log(Level.WARNING, "Error while requesting fragments: " + e.getMessage()); + this.error = e; + } } } diff --git a/src/main/java/com/limechain/utils/StringUtils.java b/src/main/java/com/limechain/utils/StringUtils.java index ba21a1768..f9f135641 100644 --- a/src/main/java/com/limechain/utils/StringUtils.java +++ b/src/main/java/com/limechain/utils/StringUtils.java @@ -3,8 +3,6 @@ import lombok.experimental.UtilityClass; import org.teavm.jso.JSBody; -import java.util.regex.Pattern; - @UtilityClass public class StringUtils { public static final String HEX_PREFIX = "0x"; @@ -29,10 +27,10 @@ public static byte[] hexToBytes(String hex) { return fromHex(hex); } - @JSBody(params = { "hex" }, script = " let bytes = [];" + - " for (let c = 0; c < hex.length; c += 2)" + - " bytes.push(parseInt(hex.substr(c, 2), 16));" + - " return bytes;") + @JSBody(params = {"hex"}, script = " let bytes = [];" + + " for (let c = 0; c < hex.length; c += 2)" + + " bytes.push(parseInt(hex.substr(c, 2), 16));" + + " return bytes;") public static native byte[] fromHex(String hex); /** @@ -49,21 +47,16 @@ public static String remove0xPrefix(String hex) { return hex; } - /** - * Converts a string to its hexadecimal representation. - * Each character of the input string is converted to its corresponding two-digit hex value. - * - * @param key the string to convert to hexadecimal - * @return the hexadecimal representation of the input string - */ - public static String toHex(String key) { - StringBuilder sb = new StringBuilder(); - char[] ch = key.toCharArray(); - for (char c : ch) { - String hexString = Integer.toHexString(c); - sb.append(hexString); + private final String hexArray = "0123456789abcdef"; + + public static String toHex(byte[] bytes) { + StringBuffer buf = new StringBuffer(); + for (int i = 0; i != bytes.length; i++) { + int v = bytes[i] & 0xff; + buf.append(hexArray.charAt(v >> 4)); + buf.append(hexArray.charAt(v & 0xf)); } - return sb.toString(); + return buf.toString(); } /** @@ -74,6 +67,6 @@ public static String toHex(String key) { * @return the "0x" prefixed hexadecimal string */ public static String toHexWithPrefix(byte[] bytes) { - return HEX_PREFIX + "Hex.toHexString(bytes)"; + return HEX_PREFIX + toHex(bytes); } } diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index a94bf6767..d2ce5d208 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -15,6 +15,8 @@ + + @@ -47,7 +49,7 @@ services: { identify: Libp2PIdentify.identify(), ping: Libp2PPing.ping(), - dht: Libp2PKadDht.kadDHT({protocol: "dot/dht/1.0.0"}), + dht: Libp2PKadDht.kadDHT({protocol: "/dot/kad"}), pubsub: ChainsafeLibp2PGossipsub.gossipsub(), } }; @@ -55,6 +57,10 @@ libp = await Libp2P.createLibp2p(test); libp.start(); + // libp.addEventListener('peer:discovery', (evt) => console.log('Discovered:', evt.detail.id.toString())) + // libp.addEventListener('peer:connect', (evt) => console.log('Connected:', evt.detail.toString())) + // libp.addEventListener('peer:disconnect', (evt) => console.log('Disconnected:', evt.detail.toString())) + // let pbStream = ItProtobufStream.pbStream; // // libp.handle('/dot/sync/warp', ({stream}) => { From 25164df6fffad2d005446fd88f84d92411cd5b5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Mon, 26 Aug 2024 13:25:03 +0300 Subject: [PATCH 02/15] chore: return string instead of array --- build.gradle.kts | 1 - .../protocol/warp/WarpSyncProtocol.java | 32 +++++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 85e0dac4d..044b01225 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,7 +30,6 @@ teavm.js { targetFileName = "fruzhin.js" } - //TODO: Debug only. Remove when doing release build teavm { js { diff --git a/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java b/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java index dc704f88b..5c85ef564 100644 --- a/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java +++ b/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java @@ -54,25 +54,21 @@ public WarpSyncResponse send(WarpSyncRequest req, String protocolId) { System.out.println("Request: " + req.getBlockHash()); final var lock = new Object(); - JSPromise> objectJSPromise = sendRequest(StringUtils.toHex(req.getBlockHash().getBytes()), protocolId); + JSPromise objectJSPromise = + sendRequest(StringUtils.toHex(req.getBlockHash().getBytes()), protocolId); objectJSPromise.then((ttt) -> { - System.out.println(ttt); - System.out.println(ttt.get(0)); - System.out.println(ttt.get(ttt.getLength() - 1)); - byte[] bytes = new byte[ttt.getLength()]; - for (int i = 0; i < ttt.getLength(); i++) { - //bytes[i] = (byte) ((int) ttt.get(i)); //fails here - } -// byte[] bytes = StringUtils.fromHex(ttt); - System.out.println("Received response: " + " " + bytes); + System.out.println("Received response: " + ttt); + System.out.println("Received response len: " + ttt.length()); + byte[] bytes = StringUtils.fromHex(ttt); + System.out.println("Received response: " + bytes); System.out.println("Received response len: " + bytes.length); - synchronized (lock) { + synchronized (lock) { // System.out.println("Received response: " + bytes.length + " " + bytes); ScaleCodecReader scaleCodecReader = new ScaleCodecReader(bytes); WarpSyncResponse responseaa = new WarpSyncResponseScaleReader().read(scaleCodecReader); - System.out.println(responseaa); + System.out.println(responseaa); // response.set(result); lock.notify(); } @@ -95,11 +91,13 @@ public WarpSyncResponse send(WarpSyncRequest req, String protocolId) { } @JSBody(params = {"blockHash", "protocolId"}, script = "return (async () => {" + - "let peer = libp.getConnections()[0].remotePeer;" + - "let stream = await ItPbStream.pbStream(await libp.dialProtocol(peer, protocolId));" + - "stream.writeLP(new Uint8Array([...blockHash.matchAll(/../g)].map(m => parseInt(m[0], 16))));" + - "return Array.from((await stream.readLP()).subarray());})()") - private static native JSPromise> sendRequest(String blockHash, String protocolId); + " let peer = libp.getConnections()[0].remotePeer;" + + " let stream = await ItPbStream.pbStream(await libp.dialProtocol(peer, protocolId));" + + " stream.writeLP(new Uint8Array([...blockHash.matchAll(/../g)].map(m => parseInt(m[0], 16))));" + + " let bytes = (await stream.readLP()).subarray();" + + " return [...bytes].map(n => n.toString(16)).join('');" + + "})()") + private static native JSPromise sendRequest(String blockHash, String protocolId); } } From 6cdded87e9eba9eebe4ef399be2db55c5fc6cec7 Mon Sep 17 00:00:00 2001 From: Yordan Atanasov Date: Mon, 26 Aug 2024 15:30:11 +0300 Subject: [PATCH 03/15] fix: add object mapper nested object functionality. --- .../java/com/limechain/utils/json/ObjectMapper.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/limechain/utils/json/ObjectMapper.java b/src/main/java/com/limechain/utils/json/ObjectMapper.java index e1f85fa0c..175d09ce5 100644 --- a/src/main/java/com/limechain/utils/json/ObjectMapper.java +++ b/src/main/java/com/limechain/utils/json/ObjectMapper.java @@ -67,18 +67,18 @@ private T convertValue(Class type, Object value) { if (type.isInstance(value)) { return (T) value; } else if (type == Integer.class || type == int.class) { - return handleWholeNumber(value, (long) Integer.MIN_VALUE, (long) Integer.MIN_VALUE); + return handleWholeNumber(value, (long) Integer.MIN_VALUE, (long) Integer.MAX_VALUE); } else if (type == Long.class || type == long.class) { return handleWholeNumber(value, Long.MIN_VALUE, Long.MAX_VALUE); } else if (type == Double.class || type == double.class) { - BigDecimal bigDecimalValue = new BigDecimal((String) value); + BigDecimal bigDecimalValue = new BigDecimal(value.toString()); double doubleValue = bigDecimalValue.doubleValue(); if (doubleValue == Double.POSITIVE_INFINITY || doubleValue == Double.NEGATIVE_INFINITY) { throw new ArithmeticException("Value out of range for Double: " + value); } return (T) Double.valueOf(doubleValue); } else if (type == BigInteger.class) { - return (T) new BigInteger((String) value); + return (T) new BigInteger(value.toString()); } else if (type == Boolean.class || type == boolean.class) { return (T) value; } else if (type == String.class) { @@ -91,6 +91,8 @@ private T convertValue(Class type, Object value) { } } else if (type.isArray()) { return (T) convertArray(type.getComponentType(), (List) value); + } else if (Map.class.isAssignableFrom(value.getClass())) { + return mapToClass(JsonUtil.stringify(value), type); } throw new RuntimeException("Unsupported field type: " + type); @@ -98,10 +100,10 @@ private T convertValue(Class type, Object value) { @SuppressWarnings("unchecked") private T handleWholeNumber(Object value, Long min, Long max) { - BigInteger bigIntValue = new BigInteger((String) value); + BigInteger bigIntValue = new BigInteger(value.toString()); if (bigIntValue.compareTo(BigInteger.valueOf(min)) < 0 || bigIntValue.compareTo(BigInteger.valueOf(max)) > 0) { - throw new ArithmeticException("Value out of range number type: " + value); + throw new ArithmeticException("Value out of range for number type: " + value); } return (T) Integer.valueOf(bigIntValue.intValue()); } From b9e826b823a089c2164aea70d7d18f3c3418f612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Mon, 26 Aug 2024 15:43:58 +0300 Subject: [PATCH 04/15] feat: working warp sync --- .../chain/lightsyncstate/AuthoritySet.java | 2 + .../chain/lightsyncstate/EpochChanges.java | 2 + .../chain/lightsyncstate/LightSyncState.java | 20 +- .../java/com/limechain/network/Network.java | 2 - .../network/protocol/warp/WarpSync.java | 18 +- .../protocol/warp/WarpSyncProtocol.java | 65 +---- .../protocol/warp/WarpSyncService.java | 2 +- .../protocol/warp/dto/BlockHeader.java | 2 + .../protocol/warp/dto/Justification.java | 2 + .../protocol/warp/dto/WarpSyncFragment.java | 2 + .../protocol/warp/dto/WarpSyncResponse.java | 2 + .../warp/scale/reader/BlockHeaderReader.java | 6 - .../limechain/storage/block/SyncState.java | 2 +- .../limechain/sync/JustificationVerifier.java | 248 +++++++++++------- .../sync/warpsync/WarpSyncMachine.java | 18 +- .../sync/warpsync/WarpSyncState.java | 99 +++---- .../action/RequestFragmentsAction.java | 37 +-- .../action/VerifyJustificationAction.java | 87 +++--- .../java/com/limechain/teavm/HttpRequest.java | 7 +- .../com/limechain/teavm/TeaVMCallback.java | 10 + 20 files changed, 324 insertions(+), 309 deletions(-) create mode 100644 src/main/java/com/limechain/teavm/TeaVMCallback.java diff --git a/src/main/java/com/limechain/chain/lightsyncstate/AuthoritySet.java b/src/main/java/com/limechain/chain/lightsyncstate/AuthoritySet.java index dd9e9a929..be1a2807c 100644 --- a/src/main/java/com/limechain/chain/lightsyncstate/AuthoritySet.java +++ b/src/main/java/com/limechain/chain/lightsyncstate/AuthoritySet.java @@ -4,11 +4,13 @@ import lombok.Getter; import lombok.Setter; import com.limechain.tuple.Pair; +import lombok.ToString; import java.math.BigInteger; @Getter @Setter +@ToString public class AuthoritySet { private Authority[] currentAuthorities; private BigInteger setId; diff --git a/src/main/java/com/limechain/chain/lightsyncstate/EpochChanges.java b/src/main/java/com/limechain/chain/lightsyncstate/EpochChanges.java index 86ed948e4..94733c24f 100644 --- a/src/main/java/com/limechain/chain/lightsyncstate/EpochChanges.java +++ b/src/main/java/com/limechain/chain/lightsyncstate/EpochChanges.java @@ -4,12 +4,14 @@ import lombok.Getter; import lombok.Setter; import com.limechain.tuple.Pair; +import lombok.ToString; import java.math.BigInteger; import java.util.Map; @Getter @Setter +@ToString public class EpochChanges { private ForkTree inner; diff --git a/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java b/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java index 4d39b6df4..fa20d171a 100644 --- a/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java +++ b/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java @@ -13,15 +13,16 @@ import java.util.Map; @Getter +@ToString public class LightSyncState { private BlockHeader finalizedBlockHeader; private EpochChanges epochChanges; private AuthoritySet grandpaAuthoritySet; - public static LightSyncState decode(Map lightSyncState) { - String header = lightSyncState.get("finalizedBlockHeader"); - String epochChanges = lightSyncState.get("babeEpochChanges"); - String grandpaAuthoritySet = lightSyncState.get("grandpaAuthoritySet"); + public static LightSyncState decode(Map lightSyncStateMap) { + String header = lightSyncStateMap.get("finalizedBlockHeader"); + String epochChanges = lightSyncStateMap.get("babeEpochChanges"); + String grandpaAuthoritySet = lightSyncStateMap.get("grandpaAuthoritySet"); if (header == null) { throw new IllegalStateException("finalizedBlockHeader is null"); @@ -34,18 +35,19 @@ public static LightSyncState decode(Map lightSyncState) { } - var state = new LightSyncState(); + LightSyncState lightSyncState = new LightSyncState(); byte[] bytes = StringUtils.hexToBytes(header); - state.finalizedBlockHeader = new BlockHeaderReader() + lightSyncState.finalizedBlockHeader = new BlockHeaderReader() .read(new ScaleCodecReader(bytes)); byte[] bytes1 = StringUtils.hexToBytes(epochChanges); - state.epochChanges = new EpochChangesReader() + lightSyncState.epochChanges = new EpochChangesReader() .read(new ScaleCodecReader(bytes1)); - state.grandpaAuthoritySet = new AuthoritySetReader() + lightSyncState.grandpaAuthoritySet = new AuthoritySetReader() .read(new ScaleCodecReader(StringUtils.hexToBytes(grandpaAuthoritySet))); - return state; + System.out.println(lightSyncState); + return lightSyncState; } } diff --git a/src/main/java/com/limechain/network/Network.java b/src/main/java/com/limechain/network/Network.java index 2c78cf58d..416cf184f 100644 --- a/src/main/java/com/limechain/network/Network.java +++ b/src/main/java/com/limechain/network/Network.java @@ -209,8 +209,6 @@ public void pingPeers() { // } // public WarpSyncResponse makeWarpSyncRequest(String blockHash) { -// if (isPeerInvalid()) return null; - return this.warpSyncService.getProtocol().warpSyncRequest( blockHash); } diff --git a/src/main/java/com/limechain/network/protocol/warp/WarpSync.java b/src/main/java/com/limechain/network/protocol/warp/WarpSync.java index 5e6075a56..71940361a 100644 --- a/src/main/java/com/limechain/network/protocol/warp/WarpSync.java +++ b/src/main/java/com/limechain/network/protocol/warp/WarpSync.java @@ -20,25 +20,15 @@ public class WarpSync extends StrictProtocolBinding { private String protocolId; - public WarpSync(String protocolId, WarpSyncProtocol protocol) { - super(protocolId/*, protocol*/); + public WarpSync(String protocolId) { + super(protocolId); this.protocolId = protocolId; } - public WarpSyncResponse warpSyncRequest(/*PeerId peer,*/ String blockHash) { -// try { -// Stream stream = dialPeer(/*peer*/); + public WarpSyncResponse warpSyncRequest(String blockHash) { WarpSyncProtocol.Sender sender = new WarpSyncProtocol.Sender(); - System.out.println("Block hash: " + blockHash); WarpSyncResponse resp = sender.warpSyncRequest(blockHash, protocolId); - log.log(Level.INFO, "Received warp sync response with " + resp/*.getFragments().length*/ + " fragments"); + log.log(Level.INFO, "Received warp sync response with " + resp.getFragments().length + " fragments"); return resp; -// } catch (ExecutionException | TimeoutException | IllegalStateException e) { -// log.log(Level.SEVERE, "Error while sending remote call request: ", e); -// throw new ExecutionFailedException(e); -// } catch (InterruptedException e) { -// Thread.currentThread().interrupt(); -// throw new ThreadInterruptedException(e); -// } } } diff --git a/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java b/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java index 5c85ef564..4d202b741 100644 --- a/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java +++ b/src/main/java/com/limechain/network/protocol/warp/WarpSyncProtocol.java @@ -3,73 +3,38 @@ import com.limechain.network.protocol.warp.dto.WarpSyncRequest; import com.limechain.network.protocol.warp.dto.WarpSyncResponse; import com.limechain.network.protocol.warp.scale.reader.WarpSyncResponseScaleReader; -import com.limechain.network.wrapper.Stream; import com.limechain.polkaj.reader.ScaleCodecReader; import com.limechain.utils.StringUtils; import org.teavm.jso.JSBody; -import org.teavm.jso.core.JSArray; import org.teavm.jso.core.JSPromise; +import org.teavm.jso.core.JSString; -import java.util.Arrays; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.atomic.AtomicReferenceArray; -public class WarpSyncProtocol /*extends ProtocolHandler*/ { - // Sizes taken from smoldot - public static final int MAX_REQUEST_SIZE = 32; - public static final int MAX_RESPONSE_SIZE = 16 * 1024 * 1024; +public class WarpSyncProtocol { public WarpSyncProtocol() { -// super(MAX_REQUEST_SIZE, MAX_RESPONSE_SIZE); } - /*@Override - protected CompletableFuture onStartInitiator(Stream stream) { - stream.pushHandler(new Leb128LengthFrameDecoder()); - stream.pushHandler(new WarpSyncResponseDecoder()); - - stream.pushHandler(new Leb128LengthFrameEncoder()); - stream.pushHandler(new ByteArrayEncoder()); - WarpSyncProtocol.Sender handler = new WarpSyncProtocol.Sender(stream); - stream.pushHandler(handler); - return CompletableFuture.completedFuture(handler); - }*/ - static class Sender implements WarpSyncController { - public static final int MAX_QUEUE_SIZE = 50; -// private final LinkedBlockingDeque> queue = -// new LinkedBlockingDeque<>(MAX_QUEUE_SIZE); public Sender() { } - // @Override - public void onMessage(Stream stream, WarpSyncResponse msg) { -// Objects.requireNonNull(queue.poll()).complete(msg); -// stream.closeWrite(); - } - @Override public WarpSyncResponse send(WarpSyncRequest req, String protocolId) { - System.out.println("Request: " + req.getBlockHash()); final var lock = new Object(); - JSPromise objectJSPromise = + AtomicReference response = new AtomicReference<>(); + JSPromise objectJSPromise = sendRequest(StringUtils.toHex(req.getBlockHash().getBytes()), protocolId); objectJSPromise.then((ttt) -> { - System.out.println("Received response: " + ttt); - System.out.println("Received response len: " + ttt.length()); - byte[] bytes = StringUtils.fromHex(ttt); - System.out.println("Received response: " + bytes); - System.out.println("Received response len: " + bytes.length); - synchronized (lock) { -// System.out.println("Received response: " + bytes.length + " " + bytes); - ScaleCodecReader scaleCodecReader = new ScaleCodecReader(bytes); - WarpSyncResponse responseaa = new WarpSyncResponseScaleReader().read(scaleCodecReader); - System.out.println(responseaa); -// response.set(result); + String str = ttt.stringValue(); + byte[] bytes = StringUtils.fromHex(str); + + response.set(bytes); lock.notify(); } return null; @@ -78,16 +43,14 @@ public WarpSyncResponse send(WarpSyncRequest req, String protocolId) { synchronized (lock) { try { lock.wait(); -// byte[] bytes = response.get(); -// System.out.println("Received response: " + /*bytes.length +*/ " " + bytes); -// ScaleCodecReader scaleCodecReader = new ScaleCodecReader(bytes); -// WarpSyncResponse responseaa = new WarpSyncResponseScaleReader().read(scaleCodecReader); -// System.out.println(responseaa); + byte[] bytes = response.get(); + ScaleCodecReader scaleCodecReader = new ScaleCodecReader(bytes); + + return new WarpSyncResponseScaleReader().read(scaleCodecReader); } catch (InterruptedException e) { throw new RuntimeException(e); } } - return null; } @JSBody(params = {"blockHash", "protocolId"}, script = "return (async () => {" + @@ -95,9 +58,9 @@ public WarpSyncResponse send(WarpSyncRequest req, String protocolId) { " let stream = await ItPbStream.pbStream(await libp.dialProtocol(peer, protocolId));" + " stream.writeLP(new Uint8Array([...blockHash.matchAll(/../g)].map(m => parseInt(m[0], 16))));" + " let bytes = (await stream.readLP()).subarray();" + - " return [...bytes].map(n => n.toString(16)).join('');" + + " return [...bytes].map(n => n.toString(16).padStart(2, '0')).join('');" + "})()") - private static native JSPromise sendRequest(String blockHash, String protocolId); + private static native JSPromise sendRequest(String blockHash, String protocolId); } } diff --git a/src/main/java/com/limechain/network/protocol/warp/WarpSyncService.java b/src/main/java/com/limechain/network/protocol/warp/WarpSyncService.java index ce14641b1..0c1c602de 100644 --- a/src/main/java/com/limechain/network/protocol/warp/WarpSyncService.java +++ b/src/main/java/com/limechain/network/protocol/warp/WarpSyncService.java @@ -5,7 +5,7 @@ public class WarpSyncService extends NetworkService { public WarpSyncService(String protocolId) { - this.protocol = new WarpSync(protocolId, new WarpSyncProtocol()); + this.protocol = new WarpSync(protocolId); } } diff --git a/src/main/java/com/limechain/network/protocol/warp/dto/BlockHeader.java b/src/main/java/com/limechain/network/protocol/warp/dto/BlockHeader.java index 46633387d..7e62c857b 100644 --- a/src/main/java/com/limechain/network/protocol/warp/dto/BlockHeader.java +++ b/src/main/java/com/limechain/network/protocol/warp/dto/BlockHeader.java @@ -5,12 +5,14 @@ import com.limechain.utils.scale.ScaleUtils; import lombok.Getter; import lombok.Setter; +import lombok.ToString; import java.math.BigInteger; import java.util.Arrays; @Setter @Getter +@ToString public class BlockHeader { // TODO: Make this const configurable public static final int BLOCK_NUMBER_SIZE = 4; diff --git a/src/main/java/com/limechain/network/protocol/warp/dto/Justification.java b/src/main/java/com/limechain/network/protocol/warp/dto/Justification.java index 2fdeeda65..c8f26c2af 100644 --- a/src/main/java/com/limechain/network/protocol/warp/dto/Justification.java +++ b/src/main/java/com/limechain/network/protocol/warp/dto/Justification.java @@ -3,6 +3,7 @@ import com.limechain.polkaj.Hash256; import lombok.Getter; import lombok.Setter; +import lombok.ToString; import lombok.extern.java.Log; import java.math.BigInteger; @@ -11,6 +12,7 @@ @Setter @Getter @Log +@ToString public class Justification { private BigInteger round; private Hash256 targetHash; diff --git a/src/main/java/com/limechain/network/protocol/warp/dto/WarpSyncFragment.java b/src/main/java/com/limechain/network/protocol/warp/dto/WarpSyncFragment.java index 829ce51ca..f6276ab2b 100644 --- a/src/main/java/com/limechain/network/protocol/warp/dto/WarpSyncFragment.java +++ b/src/main/java/com/limechain/network/protocol/warp/dto/WarpSyncFragment.java @@ -2,6 +2,7 @@ import lombok.Getter; import lombok.Setter; +import lombok.ToString; /** * Each fragment represents a change in the list of Grandpa authorities, and a list of signatures of @@ -9,6 +10,7 @@ */ @Getter @Setter +@ToString public class WarpSyncFragment { private BlockHeader header; private Justification justification; diff --git a/src/main/java/com/limechain/network/protocol/warp/dto/WarpSyncResponse.java b/src/main/java/com/limechain/network/protocol/warp/dto/WarpSyncResponse.java index fad80e347..51033db12 100644 --- a/src/main/java/com/limechain/network/protocol/warp/dto/WarpSyncResponse.java +++ b/src/main/java/com/limechain/network/protocol/warp/dto/WarpSyncResponse.java @@ -2,11 +2,13 @@ import lombok.Getter; import lombok.Setter; +import lombok.ToString; import java.util.Arrays; @Getter @Setter +@ToString public class WarpSyncResponse { private WarpSyncFragment[] fragments; private boolean isFinished; diff --git a/src/main/java/com/limechain/network/protocol/warp/scale/reader/BlockHeaderReader.java b/src/main/java/com/limechain/network/protocol/warp/scale/reader/BlockHeaderReader.java index 0c9974ea8..b40df4c4a 100644 --- a/src/main/java/com/limechain/network/protocol/warp/scale/reader/BlockHeaderReader.java +++ b/src/main/java/com/limechain/network/protocol/warp/scale/reader/BlockHeaderReader.java @@ -12,17 +12,12 @@ public class BlockHeaderReader implements ScaleReader { @Override public BlockHeader read(ScaleCodecReader reader) { BlockHeader blockHeader = new BlockHeader(); - System.out.println("BlockHeaderReader.read"); blockHeader.setParentHash(new Hash256(reader.readUint256())); - System.out.println("BlockHeaderReader.setParentHash"); // NOTE: Usage of BlockNumberReader is intentionally omitted here, // since we want this to be a compact int, not a var size int blockHeader.setBlockNumber(BigInteger.valueOf(reader.readCompactInt())); - System.out.println("BlockHeaderReader.setBlockNumber"); blockHeader.setStateRoot(new Hash256(reader.readUint256())); - System.out.println("BlockHeaderReader.setStateRoot"); blockHeader.setExtrinsicsRoot(new Hash256(reader.readUint256())); - System.out.println("BlockHeaderReader.setExtrinsicsRoot"); var digestCount = reader.readCompactInt(); HeaderDigest[] digests = new HeaderDigest[digestCount]; @@ -31,7 +26,6 @@ public BlockHeader read(ScaleCodecReader reader) { } blockHeader.setDigest(digests); - System.out.println("BlockHeaderReader.setDigest"); return blockHeader; } diff --git a/src/main/java/com/limechain/storage/block/SyncState.java b/src/main/java/com/limechain/storage/block/SyncState.java index 7022d3536..a8367d786 100644 --- a/src/main/java/com/limechain/storage/block/SyncState.java +++ b/src/main/java/com/limechain/storage/block/SyncState.java @@ -80,7 +80,7 @@ public void resetRound() { public void setLightSyncState(LightSyncState initState) { this.setId = initState.getGrandpaAuthoritySet().getSetId(); setAuthoritySet(initState.getGrandpaAuthoritySet().getCurrentAuthorities()); -// finalizeHeader(initState.getFinalizedBlockHeader()); + finalizeHeader(initState.getFinalizedBlockHeader()); } public String getStateRoot() { diff --git a/src/main/java/com/limechain/sync/JustificationVerifier.java b/src/main/java/com/limechain/sync/JustificationVerifier.java index f29f3a5eb..e790dac44 100644 --- a/src/main/java/com/limechain/sync/JustificationVerifier.java +++ b/src/main/java/com/limechain/sync/JustificationVerifier.java @@ -1,110 +1,158 @@ package com.limechain.sync; +import com.limechain.chain.lightsyncstate.Authority; +import com.limechain.network.protocol.warp.dto.Precommit; +import com.limechain.polkaj.Hash256; +import com.limechain.polkaj.Hash512; +import com.limechain.rpc.server.AppBean; +import com.limechain.storage.block.SyncState; +import com.limechain.utils.LittleEndianUtils; +import com.limechain.utils.StringUtils; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.extern.java.Log; +import org.teavm.jso.JSBody; +import org.teavm.jso.core.JSBoolean; +import org.teavm.jso.core.JSPromise; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.stream.Collectors; @Log @NoArgsConstructor(access = AccessLevel.PRIVATE) public class JustificationVerifier { -// public static boolean verify(Precommit[] precommits, BigInteger round) { -// SyncState syncState = AppBean.getBean(SyncState.class); -// Authority[] authorities = syncState.getAuthoritySet(); -// BigInteger authoritiesSetId = syncState.getSetId(); -// -// // Implementation from: https://github.com/smol-dot/smoldot -// // lib/src/finality/justification/verify.rs -// if (authorities == null || precommits.length < (authorities.length * 2 / 3) + 1) { -// log.log(Level.WARNING, "Not enough signatures"); -// return false; -// } -// -// Set seenPublicKeys = new HashSet<>(); -// Set authorityKeys = Arrays.stream(authorities) -// .map(Authority::getPublicKey) -// .map(Hash256::new) -// .collect(Collectors.toSet()); -// -// for (Precommit precommit : precommits) { -// if (!authorityKeys.contains(precommit.getAuthorityPublicKey())) { -// log.log(Level.WARNING, "Invalid Authority for precommit"); -// return false; -// } -// -// if (seenPublicKeys.contains(precommit.getAuthorityPublicKey())) { -// log.log(Level.WARNING, "Duplicated signature"); -// return false; -// } -// seenPublicKeys.add(precommit.getAuthorityPublicKey()); -// -// // TODO (from smoldot): must check signed block ancestry using `votes_ancestries` -// -// byte[] data = getDataToVerify(precommit, authoritiesSetId, round); -// -// boolean isValid = verifySignature(precommit.getAuthorityPublicKey().toString(), -// precommit.getSignature().toString(), data); -// if (!isValid) { -// log.log(Level.WARNING, "Failed to verify signature"); -// return false; -// } -// } -// log.log(Level.INFO, "All signatures were verified successfully"); -// -// // From Smoldot implementation: -// // TODO: must check that votes_ancestries doesn't contain any unused entry -// // TODO: there's also a "ghost" thing? -// -// return true; -// } -// -// private static byte[] getDataToVerify(Precommit precommit, BigInteger authoritiesSetId, BigInteger round){ -// // 1 reserved byte for data type -// // 32 reserved for target hash -// // 4 reserved for block number -// // 8 reserved for justification round -// // 8 reserved for set id -// int messageCapacity = 1 + 32 + 4 + 8 + 8; -// var messageBuffer = ByteBuffer.allocate(messageCapacity); -// messageBuffer.order(ByteOrder.LITTLE_ENDIAN); -// -// // Write message type -// messageBuffer.put((byte) 1); -// // Write target hash -// messageBuffer.put(LittleEndianUtils -// .convertBytes(StringUtils.hexToBytes(precommit.getTargetHash().toString()))); -// //Write Justification round bytes as u64 -// messageBuffer.put(LittleEndianUtils -// .bytesToFixedLength(precommit.getTargetNumber().toByteArray(), 4)); -// //Write Justification round bytes as u64 -// messageBuffer.put(LittleEndianUtils.bytesToFixedLength(round.toByteArray(), 8)); -// //Write Set Id bytes as u64 -// messageBuffer.put(LittleEndianUtils.bytesToFixedLength(authoritiesSetId.toByteArray(), 8)); -// -// //Verify message -// //Might have problems because we use the stand ED25519 instead of ED25519_zebra -// messageBuffer.rewind(); -// byte[] data = new byte[messageBuffer.remaining()]; -// messageBuffer.get(data); -// return data; -// } -// -// public static boolean verifySignature(String publicKeyHex, String signatureHex, byte[] data) { -// byte[] publicKeyBytes = Hex.decode(publicKeyHex.substring(2)); -// byte[] signatureBytes = Hex.decode(signatureHex.substring(2)); -// Ed25519PublicKeyParameters publicKeyParams = new Ed25519PublicKeyParameters(publicKeyBytes, 0); -// Ed25519Signer verifier = new Ed25519Signer(); -// verifier.init(false, publicKeyParams); -// verifier.update(data, 0, data.length); -// -// Ed25519PublicKey publicKey = -// new Ed25519PublicKey(publicKeyParams); -// Extrinsic.ED25519Signature signature = new Extrinsic.ED25519Signature(Hash512.from(signatureHex)); -// -// boolean isValid = verifier.verifySignature(signatureBytes); -// boolean result = publicKey.verify(data, signature.getValue().getBytes()); -// if (!result) { -// log.log(Level.WARNING, "Invalid signature"); -// } -// return isValid; -// } + public static boolean verify(Precommit[] precommits, BigInteger round) { + SyncState syncState = AppBean.getBean(SyncState.class); + Authority[] authorities = syncState.getAuthoritySet(); + BigInteger authoritiesSetId = syncState.getSetId(); + + // Implementation from: https://github.com/smol-dot/smoldot + // lib/src/finality/justification/verify.rs + if (authorities == null || precommits.length < (authorities.length * 2 / 3) + 1) { + log.log(Level.WARNING, "Not enough signatures"); + return false; + } + + Set seenPublicKeys = new HashSet<>(); + Set authorityKeys = + Arrays.stream(authorities).map(Authority::getPublicKey).map(Hash256::new).collect(Collectors.toSet()); + + for (Precommit precommit : precommits) { + if (!authorityKeys.contains(precommit.getAuthorityPublicKey())) { + log.log(Level.WARNING, "Invalid Authority for precommit"); + return false; + } + + if (seenPublicKeys.contains(precommit.getAuthorityPublicKey())) { + log.log(Level.WARNING, "Duplicated signature"); + return false; + } + seenPublicKeys.add(precommit.getAuthorityPublicKey()); + + // TODO (from smoldot): must check signed block ancestry using `votes_ancestries` + + byte[] data = getDataToVerify(precommit, authoritiesSetId, round); + + boolean isValid = + verifySignature(precommit.getAuthorityPublicKey().toString(), precommit.getSignature().toString(), + data); + if (!isValid) { + log.log(Level.WARNING, "Failed to verify signature"); + return false; + } + } + log.log(Level.INFO, "All signatures were verified successfully"); + + // From Smoldot implementation: + // TODO: must check that votes_ancestries doesn't contain any unused entry + // TODO: there's also a "ghost" thing? + + return true; + } + + private static byte[] getDataToVerify(Precommit precommit, BigInteger authoritiesSetId, BigInteger round) { + // 1 reserved byte for data type + // 32 reserved for target hash + // 4 reserved for block number + // 8 reserved for justification round + // 8 reserved for set id + int messageCapacity = 1 + 32 + 4 + 8 + 8; + var messageBuffer = ByteBuffer.allocate(messageCapacity); + messageBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write message type + messageBuffer.put((byte) 1); + // Write target hash + messageBuffer.put(LittleEndianUtils.convertBytes(StringUtils.hexToBytes(precommit.getTargetHash().toString()))); + //Write Justification round bytes as u64 + messageBuffer.put(LittleEndianUtils.bytesToFixedLength(precommit.getTargetNumber().toByteArray(), 4)); + //Write Justification round bytes as u64 + messageBuffer.put(LittleEndianUtils.bytesToFixedLength(round.toByteArray(), 8)); + //Write Set Id bytes as u64 + messageBuffer.put(LittleEndianUtils.bytesToFixedLength(authoritiesSetId.toByteArray(), 8)); + + //Verify message + //Might have problems because we use the stand ED25519 instead of ED25519_zebra + messageBuffer.rewind(); + byte[] data = new byte[messageBuffer.remaining()]; + messageBuffer.get(data); + return data; + } + + public static boolean verifySignature(String publicKeyHex, String signatureHex, byte[] data) { + String message = StringUtils.toHex(data); + AtomicBoolean verifier = new AtomicBoolean(false); + Object lock = new Object(); + + verifySignature(publicKeyHex, signatureHex, message).then(isValid -> { + synchronized (lock) { + verifier.set(isValid.booleanValue()); + lock.notify(); + } + return null; + }); + + synchronized (lock) { + try { + lock.wait(); + + boolean result = verifier.get(); + if (!result) { + log.log(Level.WARNING, "Invalid signature"); + } + return result; + } catch (InterruptedException e) { + log.log(Level.WARNING, "Interrupted while waiting for signature verification"); + return false; + } + } + } + + @JSBody(params = {"publicKeyHex", "signatureHex", "messageHex"}, + script = "return (async () => {" + + " const publicKeyBytes = new Uint8Array([...publicKeyHex.matchAll(/../g)].map(m => parseInt(m[0], 16)));" + + " const signatureBytes = new Uint8Array([...signatureHex.matchAll(/../g)].map(m => parseInt(m[0], 16)));" + + " const publicKey = await crypto.subtle.importKey(" + + " 'raw'," + " publicKeyBytes," + + " {" + " name: 'NODE-ED25519'," + + " namedCurve: 'ed25519'" + " }," + + " true," + " ['verify']" + " );" + + " const messageBytes = new Uint8Array([...messageHex.matchAll(/../g)].map(m => parseInt(m[0], 16)));;" + + " const isValid = await crypto.subtle.verify(" + + " {" + " name: 'NODE-ED25519'" + + " }," + " publicKey," + + " signatureBytes," + + " messageBytes" + " );" + + " return isValid;" + + + "})()") + public static native JSPromise verifySignature(String publicKeyHex, String signatureHex, + String messageHex); } diff --git a/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java b/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java index f7b0a6ed2..bfaee3f46 100644 --- a/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java +++ b/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java @@ -38,7 +38,8 @@ public class WarpSyncMachine { private final SyncState syncState; private final List onFinishCallbacks; - public WarpSyncMachine(Network network, ChainService chainService, SyncState syncState, WarpSyncState warpSyncState) { + public WarpSyncMachine(Network network, ChainService chainService, SyncState syncState, + WarpSyncState warpSyncState) { this.networkService = network; this.chainService = chainService; this.syncState = syncState; @@ -54,7 +55,6 @@ public void nextState() { } public void handleState() { - System.out.println("Warp sync action" + warpSyncAction.getClass().getSimpleName()); warpSyncAction.handle(this); } @@ -78,12 +78,12 @@ public void start() { this.warpSyncAction = new RequestFragmentsAction(initStateHash); // new Thread(() -> { -// while (this.warpSyncAction.getClass() != FinishedAction.class) { - this.handleState(); - this.nextState(); -// } -// -// finishWarpSync(); + while (this.warpSyncAction.getClass() != FinishedAction.class) { + this.handleState(); + this.nextState(); + } + + finishWarpSync(); // }).start(); } @@ -96,7 +96,7 @@ public void stop() { private void finishWarpSync() { this.warpState.setWarpSyncFinished(true); // this.networkService.handshakeBootNodes(); -// this.syncState.persistState(); + this.syncState.persistState(); log.info("Warp sync finished."); this.onFinishCallbacks.forEach(Runnable::run); } diff --git a/src/main/java/com/limechain/sync/warpsync/WarpSyncState.java b/src/main/java/com/limechain/sync/warpsync/WarpSyncState.java index e1edf1bb6..f6bd896fb 100644 --- a/src/main/java/com/limechain/sync/warpsync/WarpSyncState.java +++ b/src/main/java/com/limechain/sync/warpsync/WarpSyncState.java @@ -2,7 +2,14 @@ import com.limechain.chain.lightsyncstate.Authority; import com.limechain.network.Network; +import com.limechain.network.protocol.warp.dto.ConsensusEngine; +import com.limechain.network.protocol.warp.dto.HeaderDigest; +import com.limechain.polkaj.reader.ScaleCodecReader; import com.limechain.storage.block.SyncState; +import com.limechain.sync.warpsync.dto.AuthoritySetChange; +import com.limechain.sync.warpsync.dto.GrandpaDigestMessageType; +import com.limechain.sync.warpsync.scale.ForcedChangeReader; +import com.limechain.sync.warpsync.scale.ScheduledChangeReader; import com.limechain.tuple.Pair; import lombok.Getter; import lombok.Setter; @@ -174,51 +181,51 @@ public void handleScheduledEvents() { * @param headerDigests digest of the block header * @param blockNumber block that contains the digest */ -// public void handleAuthorityChanges(HeaderDigest[] headerDigests, BigInteger blockNumber) { -// // Update authority set and set id -// AuthoritySetChange authorityChanges; -// for (HeaderDigest digest : headerDigests) { -// if (digest.getId() == ConsensusEngine.GRANDPA) { -// ScaleCodecReader reader = new ScaleCodecReader(digest.getMessage()); -// GrandpaDigestMessageType type = GrandpaDigestMessageType.fromId(reader.readByte()); -// -// if (type == null) { -// log.log(Level.SEVERE, "Could not get grandpa message type"); -// throw new IllegalStateException("Unknown grandpa message type"); -// } -// -// switch (type) { -// case SCHEDULED_CHANGE -> { -// ScheduledChangeReader authorityChangesReader = new ScheduledChangeReader(); -// authorityChanges = authorityChangesReader.read(reader); -// scheduledAuthorityChanges -// .add(new Pair<>(blockNumber.add(authorityChanges.getDelay()), -// authorityChanges.getAuthorities())); -// return; -// } -// case FORCED_CHANGE -> { -// ForcedChangeReader authorityForcedChangesReader = new ForcedChangeReader(); -// authorityChanges = authorityForcedChangesReader.read(reader); -// scheduledAuthorityChanges -// .add(new Pair<>(blockNumber.add(authorityChanges.getDelay()), -// authorityChanges.getAuthorities())); -// return; -// } -// case ON_DISABLED -> { -// log.log(Level.SEVERE, "'ON DISABLED' grandpa message not implemented"); -// return; -// } -// case PAUSE -> { -// log.log(Level.SEVERE, "'PAUSE' grandpa message not implemented"); -// return; -// } -// case RESUME -> { -// log.log(Level.SEVERE, "'RESUME' grandpa message not implemented"); -// return; -// } -// } -// } -// } -// } + public void handleAuthorityChanges(HeaderDigest[] headerDigests, BigInteger blockNumber) { + // Update authority set and set id + AuthoritySetChange authorityChanges; + for (HeaderDigest digest : headerDigests) { + if (digest.getId() == ConsensusEngine.GRANDPA) { + ScaleCodecReader reader = new ScaleCodecReader(digest.getMessage()); + GrandpaDigestMessageType type = GrandpaDigestMessageType.fromId(reader.readByte()); + + if (type == null) { + log.log(Level.SEVERE, "Could not get grandpa message type"); + throw new IllegalStateException("Unknown grandpa message type"); + } + + switch (type) { + case SCHEDULED_CHANGE -> { + ScheduledChangeReader authorityChangesReader = new ScheduledChangeReader(); + authorityChanges = authorityChangesReader.read(reader); + scheduledAuthorityChanges + .add(new Pair<>(blockNumber.add(authorityChanges.getDelay()), + authorityChanges.getAuthorities())); + return; + } + case FORCED_CHANGE -> { + ForcedChangeReader authorityForcedChangesReader = new ForcedChangeReader(); + authorityChanges = authorityForcedChangesReader.read(reader); + scheduledAuthorityChanges + .add(new Pair<>(blockNumber.add(authorityChanges.getDelay()), + authorityChanges.getAuthorities())); + return; + } + case ON_DISABLED -> { + log.log(Level.SEVERE, "'ON DISABLED' grandpa message not implemented"); + return; + } + case PAUSE -> { + log.log(Level.SEVERE, "'PAUSE' grandpa message not implemented"); + return; + } + case RESUME -> { + log.log(Level.SEVERE, "'RESUME' grandpa message not implemented"); + return; + } + } + } + } + } } \ No newline at end of file diff --git a/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java b/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java index 671f035dc..f5df404e3 100644 --- a/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java +++ b/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java @@ -8,6 +8,9 @@ import com.limechain.sync.warpsync.WarpSyncState; import lombok.extern.java.Log; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.concurrent.LinkedBlockingQueue; import java.util.logging.Level; @Log @@ -15,7 +18,7 @@ public class RequestFragmentsAction implements WarpSyncAction { private final WarpSyncState warpSyncState; private final Hash256 blockHash; - // private WarpSyncResponse result; + private WarpSyncResponse result; private Exception error; public RequestFragmentsAction(Hash256 blockHash) { @@ -38,44 +41,32 @@ public void next(WarpSyncMachine sync) { + e.getMessage(), e.getStackTrace()); } } -// if (this.result != null) { - sync.setWarpSyncAction(new VerifyJustificationAction()); -// return; -// } + if (this.result != null) { + sync.setWarpSyncAction(new VerifyJustificationAction()); + return; + } log.log(Level.WARNING, "RequestFragmentsState.next() called without result or error set."); } @Override public void handle(WarpSyncMachine sync) { - WarpSyncResponse resp = null; -// for (int i = 0; i < sync.getNetworkService().getKademliaService().getBootNodePeerIds().size(); i++) { -// try { - resp = sync.getNetworkService().makeWarpSyncRequest(blockHash.toString()); -// break; -// } catch (Exception e) { -// if (!sync.getNetworkService().updateCurrentSelectedPeerWithBootnode(i)) { -// this.error = e; -// return; -// } -// } -// } + WarpSyncResponse resp = sync.getNetworkService().makeWarpSyncRequest(blockHash.toString()); try { if (resp == null) { throw new MissingObjectException("No response received."); } - log.log(Level.INFO, "Successfully received fragments from peer " - /* + sync.getNetworkService().getCurrentSelectedPeer()*/); + log.log(Level.INFO, "Successfully received fragments from peer"); if (resp.getFragments().length == 0) { log.log(Level.WARNING, "No fragments received."); return; } warpSyncState.setWarpSyncFragmentsFinished(resp.isFinished()); -// sync.setFragmentsQueue(new LinkedBlockingQueue<>( -// Arrays.stream(resp.getFragments()).toList()) -// ); + sync.setFragmentsQueue(new ArrayDeque<>( + Arrays.stream(resp.getFragments()).toList()) + ); -// this.result = resp; + this.result = resp; } catch (Exception e) { // TODO: Set error state, next() will use to transition to correct next state. // This error state could be either recoverable or irrecoverable. diff --git a/src/main/java/com/limechain/sync/warpsync/action/VerifyJustificationAction.java b/src/main/java/com/limechain/sync/warpsync/action/VerifyJustificationAction.java index 354b2af3a..7c07f4e4f 100644 --- a/src/main/java/com/limechain/sync/warpsync/action/VerifyJustificationAction.java +++ b/src/main/java/com/limechain/sync/warpsync/action/VerifyJustificationAction.java @@ -1,11 +1,16 @@ package com.limechain.sync.warpsync.action; +import com.limechain.exception.sync.JustificationVerificationException; +import com.limechain.network.protocol.warp.dto.WarpSyncFragment; import com.limechain.rpc.server.AppBean; import com.limechain.storage.block.SyncState; +import com.limechain.sync.JustificationVerifier; import com.limechain.sync.warpsync.WarpSyncMachine; import com.limechain.sync.warpsync.WarpSyncState; import lombok.extern.java.Log; +import java.util.logging.Level; + // VerifyJustificationState is going to be instantiated a lot of times // Maybe we can make it a singleton in order to reduce performance overhead? @Log @@ -27,51 +32,51 @@ public void next(WarpSyncMachine sync) { return; } -// if (!sync.getFragmentsQueue().isEmpty()) { -// sync.setWarpSyncAction(new VerifyJustificationAction()); -// } else if (warpSyncState.isWarpSyncFragmentsFinished()) { -// sync.setWarpSyncAction(new RuntimeDownloadAction()); -// } else { -// sync.setWarpSyncAction(new RequestFragmentsAction(syncState.getLastFinalizedBlockHash())); -// } + if (!sync.getFragmentsQueue().isEmpty()) { + sync.setWarpSyncAction(new VerifyJustificationAction()); + } else if (warpSyncState.isWarpSyncFragmentsFinished()) { + sync.setWarpSyncAction(new FinishedAction()); + } else { + sync.setWarpSyncAction(new RequestFragmentsAction(syncState.getLastFinalizedBlockHash())); + } } @Override public void handle(WarpSyncMachine sync) { -// try { -// warpSyncState.handleScheduledEvents(); -// -// WarpSyncFragment fragment = sync.getFragmentsQueue().poll(); -// log.log(Level.INFO, "Verifying justification..."); -// if (fragment == null) { -// throw new JustificationVerificationException("No such fragment"); -// } -// boolean verified = JustificationVerifier.verify( -// fragment.getJustification().getPrecommits(), -// fragment.getJustification().getRound()); -// if (!verified) { -// throw new JustificationVerificationException("Justification could not be verified."); -// } -// -// syncState.finalizeHeader(fragment.getHeader()); -// handleAuthorityChanges(fragment); -// } catch (Exception e) { -// log.log(Level.WARNING, "Error while verifying justification: " + e.getMessage()); -// this.error = e; -// } + try { + warpSyncState.handleScheduledEvents(); + + WarpSyncFragment fragment = sync.getFragmentsQueue().poll(); + log.log(Level.INFO, "Verifying justification..."); + if (fragment == null) { + throw new JustificationVerificationException("No such fragment"); + } + boolean verified = JustificationVerifier.verify( + fragment.getJustification().getPrecommits(), + fragment.getJustification().getRound()); + if (!verified) { + throw new JustificationVerificationException("Justification could not be verified."); + } + + syncState.finalizeHeader(fragment.getHeader()); + handleAuthorityChanges(fragment); + } catch (Exception e) { + log.log(Level.WARNING, "Error while verifying justification: " + e.getMessage()); + this.error = e; + } } -// private void handleAuthorityChanges(WarpSyncFragment fragment) { -// try { -// warpSyncState.handleAuthorityChanges( -// fragment.getHeader().getDigest(), -// fragment.getJustification().getTargetBlock()); -// log.log(Level.INFO, "Verified justification. Block hash is now at #" -// + syncState.getLastFinalizedBlockNumber() + ": " -// + syncState.getLastFinalizedBlockHash().toString() -// + " with state root " + syncState.getStateRoot()); -// } catch (Exception e) { -// this.error = e; -// } -// } + private void handleAuthorityChanges(WarpSyncFragment fragment) { + try { + warpSyncState.handleAuthorityChanges( + fragment.getHeader().getDigest(), + fragment.getJustification().getTargetBlock()); + log.log(Level.INFO, "Verified justification. Block hash is now at #" + + syncState.getLastFinalizedBlockNumber() + ": " + + syncState.getLastFinalizedBlockHash().toString() + + " with state root " + syncState.getStateRoot()); + } catch (Exception e) { + this.error = e; + } + } } diff --git a/src/main/java/com/limechain/teavm/HttpRequest.java b/src/main/java/com/limechain/teavm/HttpRequest.java index a8c51b2b0..4c7b984df 100644 --- a/src/main/java/com/limechain/teavm/HttpRequest.java +++ b/src/main/java/com/limechain/teavm/HttpRequest.java @@ -27,10 +27,5 @@ private static void asyncHttpRequest(String method, String url, String body, Asy } @JSBody(params = {"method", "url", "body", "callback"}, script = "return asyncHttpRequest(method, url, body, callback);") - public static native void createAsyncHttpRequest(String method, String url, String body, HttpRequestCallback callback); - - @JSFunctor - private interface HttpRequestCallback extends JSObject { - void apply(JSError error, String response); - } + public static native void createAsyncHttpRequest(String method, String url, String body, TeaVMCallback callback); } diff --git a/src/main/java/com/limechain/teavm/TeaVMCallback.java b/src/main/java/com/limechain/teavm/TeaVMCallback.java new file mode 100644 index 000000000..31d018a0f --- /dev/null +++ b/src/main/java/com/limechain/teavm/TeaVMCallback.java @@ -0,0 +1,10 @@ +package com.limechain.teavm; + +import org.teavm.jso.JSFunctor; +import org.teavm.jso.JSObject; +import org.teavm.jso.core.JSError; + +@JSFunctor +public interface TeaVMCallback extends JSObject { + void apply(JSError error, String response); +} From e831a69a2df6dae2e25d0d7f018cab49a99e0397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Wed, 28 Aug 2024 11:28:30 +0300 Subject: [PATCH 05/15] chore: working warp sync --- .../chain/lightsyncstate/Authority.java | 10 +- .../chain/lightsyncstate/LightSyncState.java | 1 - .../protocol/warp/dto/BlockHeader.java | 6 +- .../com/limechain/storage/DBConstants.java | 1 + .../limechain/storage/block/SyncState.java | 7 + .../limechain/sync/JustificationVerifier.java | 96 +++--- .../sync/warpsync/WarpSyncMachine.java | 16 +- .../java/com/limechain/utils/HashUtils.java | 23 +- src/main/webapp/index.html | 2 + src/main/webapp/js/blake2b.js | 167 +++++++++ src/main/webapp/js/ed25519.js | 319 ++++++++++++++++++ 11 files changed, 569 insertions(+), 79 deletions(-) create mode 100644 src/main/webapp/js/blake2b.js create mode 100644 src/main/webapp/js/ed25519.js diff --git a/src/main/java/com/limechain/chain/lightsyncstate/Authority.java b/src/main/java/com/limechain/chain/lightsyncstate/Authority.java index e612d2efd..65cc3ea10 100644 --- a/src/main/java/com/limechain/chain/lightsyncstate/Authority.java +++ b/src/main/java/com/limechain/chain/lightsyncstate/Authority.java @@ -1,14 +1,20 @@ package com.limechain.chain.lightsyncstate; +import com.limechain.teavm.annotation.Reflectable; import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import java.io.Serializable; import java.math.BigInteger; @Getter +@Setter +@Reflectable @AllArgsConstructor +@NoArgsConstructor public class Authority implements Serializable { - private final byte[] publicKey; - private final BigInteger weight; + private byte[] publicKey; + private BigInteger weight; } diff --git a/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java b/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java index fa20d171a..a5025658e 100644 --- a/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java +++ b/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java @@ -47,7 +47,6 @@ public static LightSyncState decode(Map lightSyncStateMap) { lightSyncState.grandpaAuthoritySet = new AuthoritySetReader() .read(new ScaleCodecReader(StringUtils.hexToBytes(grandpaAuthoritySet))); - System.out.println(lightSyncState); return lightSyncState; } } diff --git a/src/main/java/com/limechain/network/protocol/warp/dto/BlockHeader.java b/src/main/java/com/limechain/network/protocol/warp/dto/BlockHeader.java index 7e62c857b..cbe856a35 100644 --- a/src/main/java/com/limechain/network/protocol/warp/dto/BlockHeader.java +++ b/src/main/java/com/limechain/network/protocol/warp/dto/BlockHeader.java @@ -2,10 +2,13 @@ import com.limechain.network.protocol.blockannounce.scale.BlockHeaderScaleWriter; import com.limechain.polkaj.Hash256; +import com.limechain.utils.HashUtils; +import com.limechain.utils.StringUtils; import com.limechain.utils.scale.ScaleUtils; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.teavm.jso.core.JSString; import java.math.BigInteger; import java.util.Arrays; @@ -37,6 +40,7 @@ public String toString() { public Hash256 getHash() { byte[] scaleEncoded = ScaleUtils.Encode.encode(BlockHeaderScaleWriter.getInstance(), this); - return null;//new Hash256(HashUtils.hashWithBlake2b(scaleEncoded)); + JSString jsString = HashUtils.hashWithBlake2b(StringUtils.toHex(scaleEncoded)); + return new Hash256(StringUtils.hexToBytes(jsString.stringValue())); } } diff --git a/src/main/java/com/limechain/storage/DBConstants.java b/src/main/java/com/limechain/storage/DBConstants.java index 3ef7b2d35..9fd4bb725 100644 --- a/src/main/java/com/limechain/storage/DBConstants.java +++ b/src/main/java/com/limechain/storage/DBConstants.java @@ -21,6 +21,7 @@ public class DBConstants { public static final String AUTHORITY_SET = "ss::authoritySet"; public static final String LATEST_ROUND = "ss::latestRound"; public static final String SET_ID = "ss::setId"; + public static final String STATE_ROOT = "ss::stateRoot"; // } diff --git a/src/main/java/com/limechain/storage/block/SyncState.java b/src/main/java/com/limechain/storage/block/SyncState.java index a8367d786..e18ed5106 100644 --- a/src/main/java/com/limechain/storage/block/SyncState.java +++ b/src/main/java/com/limechain/storage/block/SyncState.java @@ -24,6 +24,7 @@ public class SyncState { private final BigInteger startingBlock; private final Hash256 genesisBlockHash; private Hash256 lastFinalizedBlockHash; + private Hash256 stateRoot; @Setter private Authority[] authoritySet; private BigInteger latestRound; @@ -41,6 +42,8 @@ private void loadState() { DBConstants.LAST_FINALIZED_BLOCK_NUMBER, BigInteger.class).orElse(BigInteger.ZERO); this.lastFinalizedBlockHash = new Hash256(LocalStorage.find( DBConstants.LAST_FINALIZED_BLOCK_HASH, byte[].class).orElse(genesisBlockHash.getBytes())); + byte[] stateRootBytes = LocalStorage.find(DBConstants.STATE_ROOT, byte[].class).orElse(null); + this.stateRoot = stateRootBytes != null ? new Hash256(stateRootBytes) : null; this.authoritySet = LocalStorage.find(DBConstants.AUTHORITY_SET, Authority[].class).orElse(new Authority[0]); this.latestRound = LocalStorage.find(DBConstants.LATEST_ROUND, BigInteger.class).orElse(BigInteger.ONE); this.setId = LocalStorage.find(DBConstants.SET_ID, BigInteger.class).orElse(BigInteger.ZERO); @@ -52,17 +55,21 @@ public void persistState() { LocalStorage.save(DBConstants.AUTHORITY_SET, authoritySet); LocalStorage.save(DBConstants.LATEST_ROUND, latestRound); LocalStorage.save(DBConstants.SET_ID, setId); + LocalStorage.save(DBConstants.STATE_ROOT, stateRoot.getBytes()); } public void finalizeHeader(BlockHeader header) { this.lastFinalizedBlockNumber = header.getBlockNumber(); this.lastFinalizedBlockHash = header.getHash(); + this.stateRoot = header.getStateRoot(); } public void finalizedCommitMessage(CommitMessage commitMessage) { try { this.lastFinalizedBlockHash = commitMessage.getVote().getBlockHash(); this.lastFinalizedBlockNumber = commitMessage.getVote().getBlockNumber(); + this.setId = commitMessage.getSetId(); + this.latestRound = commitMessage.getRoundNumber(); } catch (HeaderNotFoundException ignored) { log.fine("Received commit message for a block that is not in the block store"); } diff --git a/src/main/java/com/limechain/sync/JustificationVerifier.java b/src/main/java/com/limechain/sync/JustificationVerifier.java index e790dac44..497a9203a 100644 --- a/src/main/java/com/limechain/sync/JustificationVerifier.java +++ b/src/main/java/com/limechain/sync/JustificationVerifier.java @@ -3,7 +3,6 @@ import com.limechain.chain.lightsyncstate.Authority; import com.limechain.network.protocol.warp.dto.Precommit; import com.limechain.polkaj.Hash256; -import com.limechain.polkaj.Hash512; import com.limechain.rpc.server.AppBean; import com.limechain.storage.block.SyncState; import com.limechain.utils.LittleEndianUtils; @@ -18,6 +17,14 @@ import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.security.InvalidKeyException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.X509EncodedKeySpec; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -60,9 +67,11 @@ public static boolean verify(Precommit[] precommits, BigInteger round) { byte[] data = getDataToVerify(precommit, authoritiesSetId, round); - boolean isValid = - verifySignature(precommit.getAuthorityPublicKey().toString(), precommit.getSignature().toString(), - data); + boolean isValid = verifySignature( + StringUtils.toHex(precommit.getAuthorityPublicKey().getBytes()), + StringUtils.toHex(precommit.getSignature().getBytes()), + StringUtils.toHex(data)); + if (!isValid) { log.log(Level.WARNING, "Failed to verify signature"); return false; @@ -77,6 +86,32 @@ public static boolean verify(Precommit[] precommits, BigInteger round) { return true; } + private static boolean verifySignature(String publicKeyHex, String signatureHex, + String messageHex) { + JSPromise verifyAsync = verifyAsync(publicKeyHex, signatureHex, messageHex); + Object lock = new Object(); + AtomicBoolean valid = new AtomicBoolean(false); + + verifyAsync.then((isValid) -> { + synchronized (lock) { + valid.set(isValid.booleanValue()); + lock.notify(); + } + return null; + }); + + boolean isValid; + synchronized (lock) { + try { + lock.wait(); + isValid = valid.get(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return isValid; + } + private static byte[] getDataToVerify(Precommit precommit, BigInteger authoritiesSetId, BigInteger round) { // 1 reserved byte for data type // 32 reserved for target hash @@ -106,53 +141,8 @@ private static byte[] getDataToVerify(Precommit precommit, BigInteger authoritie return data; } - public static boolean verifySignature(String publicKeyHex, String signatureHex, byte[] data) { - String message = StringUtils.toHex(data); - AtomicBoolean verifier = new AtomicBoolean(false); - Object lock = new Object(); - - verifySignature(publicKeyHex, signatureHex, message).then(isValid -> { - synchronized (lock) { - verifier.set(isValid.booleanValue()); - lock.notify(); - } - return null; - }); - - synchronized (lock) { - try { - lock.wait(); - - boolean result = verifier.get(); - if (!result) { - log.log(Level.WARNING, "Invalid signature"); - } - return result; - } catch (InterruptedException e) { - log.log(Level.WARNING, "Interrupted while waiting for signature verification"); - return false; - } - } - } - - @JSBody(params = {"publicKeyHex", "signatureHex", "messageHex"}, - script = "return (async () => {" + - " const publicKeyBytes = new Uint8Array([...publicKeyHex.matchAll(/../g)].map(m => parseInt(m[0], 16)));" + - " const signatureBytes = new Uint8Array([...signatureHex.matchAll(/../g)].map(m => parseInt(m[0], 16)));" + - " const publicKey = await crypto.subtle.importKey(" + - " 'raw'," + " publicKeyBytes," + - " {" + " name: 'NODE-ED25519'," + - " namedCurve: 'ed25519'" + " }," + - " true," + " ['verify']" + " );" + - " const messageBytes = new Uint8Array([...messageHex.matchAll(/../g)].map(m => parseInt(m[0], 16)));;" + - " const isValid = await crypto.subtle.verify(" + - " {" + " name: 'NODE-ED25519'" + - " }," + " publicKey," + - " signatureBytes," + - " messageBytes" + " );" + - " return isValid;" + - - "})()") - public static native JSPromise verifySignature(String publicKeyHex, String signatureHex, - String messageHex); + @JSBody(params = {"publicKeyHex", "signatureHex", + "messageHex"}, script = "return verifyAsync(signatureHex, messageHex, publicKeyHex);") + public static native JSPromise verifyAsync(String publicKeyHex, String signatureHex, + String messageHex); } diff --git a/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java b/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java index bfaee3f46..c281bf243 100644 --- a/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java +++ b/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java @@ -63,17 +63,19 @@ public boolean isSyncing() { } public void start() { - if (this.chainService.getChainSpec().getLightSyncState() != null) { - LightSyncState initState = LightSyncState.decode(this.chainService.getChainSpec().getLightSyncState()); - if (this.syncState.getLastFinalizedBlockNumber() - .compareTo(initState.getFinalizedBlockHeader().getBlockNumber()) < 0) { - this.syncState.setLightSyncState(initState); - } + LightSyncState initState = LightSyncState.decode(this.chainService.getChainSpec().getLightSyncState()); + + if (this.syncState.getLastFinalizedBlockNumber() + .compareTo(initState.getFinalizedBlockHeader().getBlockNumber()) < 0) { + this.syncState.setLightSyncState(initState); } + System.out.println(this.syncState.getLastFinalizedBlockHash()); + System.out.println(this.syncState.getLastFinalizedBlockNumber()); + final Hash256 initStateHash = this.syncState.getLastFinalizedBlockHash(); // Always start with requesting fragments - log.log(Level.INFO, "Requesting fragments... " + initStateHash.toString()); + log.log(Level.INFO, "Requesting fragments... " + initStateHash); this.networkService.updateCurrentSelectedPeerWithNextBootnode(); this.warpSyncAction = new RequestFragmentsAction(initStateHash); diff --git a/src/main/java/com/limechain/utils/HashUtils.java b/src/main/java/com/limechain/utils/HashUtils.java index 2989d78c9..e2a895553 100644 --- a/src/main/java/com/limechain/utils/HashUtils.java +++ b/src/main/java/com/limechain/utils/HashUtils.java @@ -1,22 +1,15 @@ package com.limechain.utils; +import com.limechain.polkaj.Hash256; import lombok.experimental.UtilityClass; +import org.teavm.jso.JSBody; +import org.teavm.jso.core.JSString; @UtilityClass public class HashUtils { -// public static final int HASH256_HASH_LENGTH = Hash256.SIZE_BYTES * Byte.SIZE; -// -// /** -// * Conducts a 256-bit Blake2b hash. -// * @param input the data to be hashed. -// * @return byte array containing the 256-bit hash result. -// */ -// public static byte[] hashWithBlake2b(byte[] input) { -// Blake2bDigest digest = new Blake2bDigest(HASH256_HASH_LENGTH); -// digest.reset(); -// digest.update(input, 0, input.length); -// byte[] hash = new byte[digest.getDigestSize()]; -// digest.doFinal(hash, 0); -// return hash; -// } + + @JSBody(params = {"inputHex"}, script = "{" + + "let bytes = new Uint8Array([...inputHex.matchAll(/../g)].map(m => parseInt(m[0], 16)));" + + "return Blake2b.hash(bytes,undefined,32);" + "}") + public static native JSString hashWithBlake2b(String inputHex); } diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index d2ce5d208..517da320f 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -18,6 +18,8 @@ + + diff --git a/src/main/webapp/js/blake2b.js b/src/main/webapp/js/blake2b.js new file mode 100644 index 000000000..6b5de4354 --- /dev/null +++ b/src/main/webapp/js/blake2b.js @@ -0,0 +1,167 @@ +var Blake2b = { + v: new Uint32Array(32), + m: new Uint32Array(32), + BLAKE2B_IV32: new Uint32Array([4089235720, 1779033703, 2227873595, 3144134277, 4271175723, 1013904242, 1595750129, 2773480762, 2917565137, 1359893119, 725511199, 2600822924, 4215389547, 528734635, 327033209, 1541459225]), + parameterBlock: new Uint8Array(64).fill(0), + SIGMA82: new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3, 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4, 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8, 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13, 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9, 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11, 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10, 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5, 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3].map(function (x) { + return x * 2; + })), + ADD64AA(v2, a, b) { + const o0 = v2[a] + v2[b]; + let o1 = v2[a + 1] + v2[b + 1]; + if (o0 >= 4294967296) o1++; + v2[a] = o0; + v2[a + 1] = o1; + }, + ADD64AC(v2, a, b0, b1) { + let o0 = v2[a] + b0; + if (b0 < 0) o0 += 4294967296; + let o1 = v2[a + 1] + b1; + if (o0 >= 4294967296) o1++; + v2[a] = o0; + v2[a + 1] = o1; + }, + B2B_GET32(arr, i) { + return arr[i] ^ arr[i + 1] << 8 ^ arr[i + 2] << 16 ^ arr[i + 3] << 24; + }, + B2B_G(a, b, c, d, ix, iy) { + const x0 = (this.m)[ix]; + const x1 = (this.m)[ix + 1]; + const y0 = (this.m)[iy]; + const y1 = (this.m)[iy + 1]; + this.ADD64AA(this.v, a, b); + this.ADD64AC(this.v, a, x0, x1); + let xor0 = (this.v)[d] ^ (this.v)[a]; + let xor1 = (this.v)[d + 1] ^ (this.v)[a + 1]; + (this.v)[d] = xor1; + (this.v)[d + 1] = xor0; + this.ADD64AA(this.v, c, d); + xor0 = (this.v)[b] ^ (this.v)[c]; + xor1 = (this.v)[b + 1] ^ (this.v)[c + 1]; + (this.v)[b] = xor0 >>> 24 ^ xor1 << 8; + (this.v)[b + 1] = xor1 >>> 24 ^ xor0 << 8; + this.ADD64AA(this.v, a, b); + this.ADD64AC(this.v, a, y0, y1); + xor0 = (this.v)[d] ^ (this.v)[a]; + xor1 = (this.v)[d + 1] ^ (this.v)[a + 1]; + (this.v)[d] = xor0 >>> 16 ^ xor1 << 16; + (this.v)[d + 1] = xor1 >>> 16 ^ xor0 << 16; + this.ADD64AA(this.v, c, d); + xor0 = (this.v)[b] ^ (this.v)[c]; + xor1 = (this.v)[b + 1] ^ (this.v)[c + 1]; + (this.v)[b] = xor1 >>> 31 ^ xor0 << 1; + (this.v)[b + 1] = xor0 >>> 31 ^ xor1 << 1; + }, + blake2bCompress(ctx, last) { + let i = 0; + for (i = 0; i < 16; i++) { + (this.v)[i] = ctx.h[i]; + (this.v)[i + 16] = (this.BLAKE2B_IV32)[i]; + } + (this.v)[24] = (this.v)[24] ^ ctx.t; + (this.v)[25] = (this.v)[25] ^ ctx.t / 4294967296; + if (last) { + (this.v)[28] = ~(this.v)[28]; + (this.v)[29] = ~(this.v)[29]; + } + for (i = 0; i < 32; i++) { + (this.m)[i] = this.B2B_GET32(ctx.b, 4 * i); + } + for (i = 0; i < 12; i++) { + this.B2B_G(0, 8, 16, 24, (this.SIGMA82)[i * 16 + 0], (this.SIGMA82)[i * 16 + 1]); + this.B2B_G(2, 10, 18, 26, (this.SIGMA82)[i * 16 + 2], (this.SIGMA82)[i * 16 + 3]); + this.B2B_G(4, 12, 20, 28, (this.SIGMA82)[i * 16 + 4], (this.SIGMA82)[i * 16 + 5]); + this.B2B_G(6, 14, 22, 30, (this.SIGMA82)[i * 16 + 6], (this.SIGMA82)[i * 16 + 7]); + this.B2B_G(0, 10, 20, 30, (this.SIGMA82)[i * 16 + 8], (this.SIGMA82)[i * 16 + 9]); + this.B2B_G(2, 12, 22, 24, (this.SIGMA82)[i * 16 + 10], (this.SIGMA82)[i * 16 + 11]); + this.B2B_G(4, 14, 16, 26, (this.SIGMA82)[i * 16 + 12], (this.SIGMA82)[i * 16 + 13]); + this.B2B_G(6, 8, 18, 28, (this.SIGMA82)[i * 16 + 14], (this.SIGMA82)[i * 16 + 15]); + } + for (i = 0; i < 16; i++) { + ctx.h[i] = ctx.h[i] ^ (this.v)[i] ^ (this.v)[i + 16]; + } + }, + blake2bInit(outlen, key, salt, personal) { + if (outlen === 0 || outlen > 64) { + throw new Error("Illegal output length, expected 0 < length <= 64"); + } + if (key && key.length > 64) { + throw new Error("Illegal key, expected Uint8Array with 0 < length <= 64"); + } + if (salt && salt.length !== 16) { + throw new Error("Illegal salt, expected Uint8Array with length is 16"); + } + if (personal && personal.length !== 16) { + throw new Error("Illegal personal, expected Uint8Array with length is 16"); + } + const ctx = { + b: new Uint8Array(128), h: new Uint32Array(16), t: 0, c: 0, outlen + }; + this.parameterBlock.fill(0); + (this.parameterBlock)[0] = outlen; + if (key) (this.parameterBlock)[1] = key.length; + (this.parameterBlock)[2] = 1; + (this.parameterBlock)[3] = 1; + if (salt) this.parameterBlock.set(salt, 32); + if (personal) this.parameterBlock.set(personal, 48); + for (let i = 0; i < 16; i++) { + ctx.h[i] = (this.BLAKE2B_IV32)[i] ^ this.B2B_GET32(this.parameterBlock, i * 4); + } + if (key) { + this.blake2bUpdate(ctx, key); + ctx.c = 128; + } + return ctx; + }, + blake2bUpdate(ctx, input) { + for (let i = 0; i < input.length; i++) { + if (ctx.c === 128) { + ctx.t += ctx.c; + this.blake2bCompress(ctx, false); + ctx.c = 0; + } + ctx.b[ctx.c++] = input[i]; + } + }, + blake2bFinal(ctx) { + ctx.t += ctx.c; + while (ctx.c < 128) { + ctx.b[ctx.c++] = 0; + } + this.blake2bCompress(ctx, true); + const out = new Uint8Array(ctx.outlen); + for (let i = 0; i < ctx.outlen; i++) { + out[i] = ctx.h[i >> 2] >> 8 * (i & 3); + } + return out; + }, + blake2bStart(input, key, outlen, salt, personal) { + outlen = outlen || 64; + const ctx = this.blake2bInit(outlen, key, this.normalizeInput(salt), this.normalizeInput(personal)); + this.blake2bUpdate(ctx, this.normalizeInput(input)); + return this.blake2bFinal(ctx); + }, + normalizeInput(input) { + let ret; + if (input instanceof Uint8Array) { + ret = input; + } else if (typeof input === "string") { + const encoder = new TextEncoder; + ret = encoder.encode(input); + } else { + throw new Error("Input must be an string, Buffer or Uint8Array"); + } + return ret; + }, + toHex(bytes) { + return Array.prototype.map.call(bytes, function (n) { + return (n < 16 ? "0" : "") + n.toString(16); + }).join(""); + }, + hash(message = "", secret = undefined, length = 64, salt = new Uint8Array(16), personal = new Uint8Array(16)) { + if (secret?.length === 0) secret = undefined; + if (typeof secret === "string") secret = new TextEncoder().encode(secret); + const output = this.blake2bStart(message, secret, length, salt, personal); + return this.toHex(output); + } +} \ No newline at end of file diff --git a/src/main/webapp/js/ed25519.js b/src/main/webapp/js/ed25519.js new file mode 100644 index 000000000..a640b1019 --- /dev/null +++ b/src/main/webapp/js/ed25519.js @@ -0,0 +1,319 @@ +const P = 2n ** 255n - 19n; // ed25519 is twisted edwards curve +const N = 2n ** 252n + 27742317777372353535851937790883648493n; // curve's (group) order +const Gx = 0x216936d3cd6e53fec0a4e231fdd6dc5c692cc7609525a7b2c9562d608f25d51an; // base point x +const Gy = 0x6666666666666666666666666666666666666666666666666666666666666658n; // base point y +const CURVE = { + a: -1n, // where a=-1, d = -(121665/121666) == -(121665 * inv(121666)) mod P + d: 37095705934669439343138083508754565189542113879843219016388785533085940283555n, + p: P, n: N, h: 8, Gx, Gy // field prime, curve (group) order, cofactor +}; +const err = (m = '') => { throw new Error(m); }; // error helper, messes-up stack trace +const str = (s) => typeof s === 'string'; // is string +const isu8 = (a) => (a instanceof Uint8Array || + (a != null && typeof a === 'object' && a.constructor.name === 'Uint8Array')); +const au8 = (a, l) => // is Uint8Array (of specific length) + !isu8(a) || (typeof l === 'number' && l > 0 && a.length !== l) ? + err('Uint8Array of valid length expected') : a; +const u8n = (data) => new Uint8Array(data); // creates Uint8Array +const toU8 = (a, len) => au8(str(a) ? h2b(a) : u8n(au8(a)), len); // norm(hex/u8a) to u8a +const mod = (a, b = P) => { let r = a % b; return r >= 0n ? r : b + r; }; // mod division +const isPoint = (p) => (p instanceof Point ? p : err('Point expected')); // is xyzt point + +class Point { + constructor(ex, ey, ez, et) { + this.ex = ex; + this.ey = ey; + this.ez = ez; + this.et = et; + } + static fromHex(hex, zip215 = false) { + const { d } = CURVE; + hex = toU8(hex, 32); + const normed = hex.slice(); // copy the array to not mess it up + const lastByte = hex[31]; + normed[31] = lastByte & ~0x80; // adjust first LE byte = last BE byte + const y = b2n_LE(normed); // decode as little-endian, convert to num + if (zip215 && !(0n <= y && y < 2n ** 256n)) + err('bad y coord 1'); // zip215=true [1..2^256-1] + if (!zip215 && !(0n <= y && y < P)) + err('bad y coord 2'); // zip215=false [1..P-1] + const y2 = mod(y * y); // y² + const u = mod(y2 - 1n); // u=y²-1 + const v = mod(d * y2 + 1n); // v=dy²+1 + let { isValid, value: x } = uvRatio(u, v); // (uv³)(uv⁷)^(p-5)/8; square root + if (!isValid) + err('bad y coordinate 3'); // not square root: bad point + const isXOdd = (x & 1n) === 1n; // adjust sign of x coordinate + const isLastByteOdd = (lastByte & 0x80) !== 0; // x_0, last bit + if (!zip215 && x === 0n && isLastByteOdd) + err('bad y coord 3'); // x=0 and x_0 = 1 + if (isLastByteOdd !== isXOdd) + x = mod(-x); + return new Point(x, y, 1n, mod(x * y)); // Z=1, T=xy + } + equals(other) { + const { ex: X1, ey: Y1, ez: Z1 } = this; + const { ex: X2, ey: Y2, ez: Z2 } = isPoint(other); // isPoint() checks class equality + const X1Z2 = mod(X1 * Z2), X2Z1 = mod(X2 * Z1); + const Y1Z2 = mod(Y1 * Z2), Y2Z1 = mod(Y2 * Z1); + return X1Z2 === X2Z1 && Y1Z2 === Y2Z1; + } + is0() { return this.equals(I); } + negate() { + return new Point(mod(-this.ex), this.ey, this.ez, mod(-this.et)); + } + double() { + const { ex: X1, ey: Y1, ez: Z1 } = this; // Cost: 4M + 4S + 1*a + 6add + 1*2 + const { a } = CURVE; // https://hyperelliptic.org/EFD/g1p/auto-twisted-extended.html#doubling-dbl-2008-hwcd + const A = mod(X1 * X1); + const B = mod(Y1 * Y1); + const C = mod(2n * mod(Z1 * Z1)); + const D = mod(a * A); + const x1y1 = X1 + Y1; + const E = mod(mod(x1y1 * x1y1) - A - B); + const G = D + B; + const F = G - C; + const H = D - B; + const X3 = mod(E * F); + const Y3 = mod(G * H); + const T3 = mod(E * H); + const Z3 = mod(F * G); + return new Point(X3, Y3, Z3, T3); + } + add(other) { + const { ex: X1, ey: Y1, ez: Z1, et: T1 } = this; // Cost: 8M + 1*k + 8add + 1*2. + const { ex: X2, ey: Y2, ez: Z2, et: T2 } = isPoint(other); // doesn't check if other on-curve + const { a, d } = CURVE; // http://hyperelliptic.org/EFD/g1p/auto-twisted-extended-1.html#addition-add-2008-hwcd-3 + const A = mod(X1 * X2); + const B = mod(Y1 * Y2); + const C = mod(T1 * d * T2); + const D = mod(Z1 * Z2); + const E = mod((X1 + Y1) * (X2 + Y2) - A - B); + const F = mod(D - C); + const G = mod(D + C); + const H = mod(B - a * A); + const X3 = mod(E * F); + const Y3 = mod(G * H); + const T3 = mod(E * H); + const Z3 = mod(F * G); + return new Point(X3, Y3, Z3, T3); + } + mul(n, safe = true) { + if (n === 0n) + return safe === true ? err('cannot multiply by 0') : I; + if (!(typeof n === 'bigint' && 0n < n && n < N)) + err('invalid scalar, must be < L'); + if (!safe && this.is0() || n === 1n) + return this; // safe=true bans 0. safe=false allows 0. + if (this.equals(G)) + return wNAF(n).p; // use wNAF precomputes for base points + let p = I, f = G; // init result point & fake point + for (let d = this; n > 0n; d = d.double(), n >>= 1n) { // double-and-add ladder + if (n & 1n) + p = p.add(d); // if bit is present, add to point + else if (safe) + f = f.add(d); // if not, add to fake for timing safety + } + return p; + } + clearCofactor() { return this.mul(BigInt(CURVE.h), false); } // multiply by cofactor + isSmallOrder() { return this.clearCofactor().is0(); } // check if P is small order + toAffine() { + const { ex: x, ey: y, ez: z } = this; // (x, y, z, t) ∋ (x=x/z, y=y/z, t=xy) + if (this.equals(I)) + return { x: 0n, y: 1n }; // fast-path for zero point + const iz = invert(z); // z^-1: invert z + if (mod(z * iz) !== 1n) + err('invalid inverse'); // (z * z^-1) must be 1, otherwise bad math + return { x: mod(x * iz), y: mod(y * iz) }; // x = x*z^-1; y = y*z^-1 + } + toRawBytes() { + const { x, y } = this.toAffine(); // convert to affine 2d point + const b = n2b_32LE(y); // encode number to 32 bytes + b[31] |= x & 1n ? 0x80 : 0; // store sign in first LE byte + return b; + } +} +Point.BASE = new Point(Gx, Gy, 1n, mod(Gx * Gy)); // Generator / Base point +Point.ZERO = new Point(0n, 1n, 1n, 0n); // Identity / Zero point +const { BASE: G, ZERO: I } = Point; // Generator, identity points +const padh = (num, pad) => num.toString(16).padStart(pad, '0'); +const b2h = (b) => Array.from(b).map(e => padh(e, 2)).join(''); // bytes to hex +const h2b = (hex) => { + const l = hex.length; // error if not string, + if (!str(hex) || l % 2) + err('hex invalid 1'); // or has odd length like 3, 5. + const arr = u8n(l / 2); // create result array + for (let i = 0; i < arr.length; i++) { + const j = i * 2; + const h = hex.slice(j, j + 2); // hexByte. slice is faster than substr + const b = Number.parseInt(h, 16); // byte, created from string part + if (Number.isNaN(b) || b < 0) + err('hex invalid 2'); // byte must be valid 0 <= byte < 256 + arr[i] = b; + } + return arr; +}; + +const n2b_32LE = (num) => h2b(padh(num, 32 * 2)).reverse(); // number to bytes LE +const b2n_LE = (b) => BigInt('0x' + b2h(u8n(au8(b)).reverse())); // bytes LE to num +const concatB = (...arrs) => { + const r = u8n(arrs.reduce((sum, a) => sum + au8(a).length, 0)); // create u8a of summed length + let pad = 0; // walk through each array, + arrs.forEach(a => { r.set(a, pad); pad += a.length; }); // ensure they have proper type + return r; +}; +const invert = (num, md = P) => { + if (num === 0n || md <= 0n) + err('no inverse n=' + num + ' mod=' + md); // no neg exponent for now + let a = mod(num, md), b = md, x = 0n, y = 1n, u = 1n, v = 0n; + while (a !== 0n) { // uses euclidean gcd algorithm + const q = b / a, r = b % a; // not constant-time + const m = x - u * q, n = y - v * q; + b = a, a = r, x = u, y = v, u = m, v = n; + } + return b === 1n ? mod(x, md) : err('no inverse'); // b is gcd at this point +}; +const pow2 = (x, power) => { + let r = x; + while (power-- > 0n) { + r *= r; + r %= P; + } + return r; +}; +const pow_2_252_3 = (x) => { + const x2 = (x * x) % P; // x^2, bits 1 + const b2 = (x2 * x) % P; // x^3, bits 11 + const b4 = (pow2(b2, 2n) * b2) % P; // x^(2^4-1), bits 1111 + const b5 = (pow2(b4, 1n) * x) % P; // x^(2^5-1), bits 11111 + const b10 = (pow2(b5, 5n) * b5) % P; // x^(2^10) + const b20 = (pow2(b10, 10n) * b10) % P; // x^(2^20) + const b40 = (pow2(b20, 20n) * b20) % P; // x^(2^40) + const b80 = (pow2(b40, 40n) * b40) % P; // x^(2^80) + const b160 = (pow2(b80, 80n) * b80) % P; // x^(2^160) + const b240 = (pow2(b160, 80n) * b80) % P; // x^(2^240) + const b250 = (pow2(b240, 10n) * b10) % P; // x^(2^250) + const pow_p_5_8 = (pow2(b250, 2n) * x) % P; // < To pow to (p+3)/8, multiply it by x. + return { pow_p_5_8, b2 }; +}; +const RM1 = 19681161376707505956807079304988542015446066515923890162744021073123829784752n; // √-1 +const uvRatio = (u, v) => { + const v3 = mod(v * v * v); // v³ + const v7 = mod(v3 * v3 * v); // v⁷ + const pow = pow_2_252_3(u * v7).pow_p_5_8; // (uv⁷)^(p-5)/8 + let x = mod(u * v3 * pow); // (uv³)(uv⁷)^(p-5)/8 + const vx2 = mod(v * x * x); // vx² + const root1 = x; // First root candidate + const root2 = mod(x * RM1); // Second root candidate; RM1 is √-1 + const useRoot1 = vx2 === u; // If vx² = u (mod p), x is a square root + const useRoot2 = vx2 === mod(-u); // If vx² = -u, set x <-- x * 2^((p-1)/4) + const noRoot = vx2 === mod(-u * RM1); // There is no valid root, vx² = -u√-1 + if (useRoot1) + x = root1; + if (useRoot2 || noRoot) + x = root2; // We return root2 anyway, for const-time + if ((mod(x) & 1n) === 1n) + x = mod(-x); // edIsNegative + return { isValid: useRoot1 || useRoot2, value: x }; +}; +const modL_LE = (hash) => mod(b2n_LE(hash), N); // modulo L; but little-endian +let _shaS; +const sha512a = (...m) => etc.sha512Async(...m); // Async SHA512 +const sha512s = (...m) => // Sync SHA512, not set by default + typeof _shaS === 'function' ? _shaS(...m) : err('etc.sha512Sync not set'); + +function hashFinish(asynchronous, res) { + if (asynchronous) + return sha512a(res.hashable).then(res.finish); + return res.finish(sha512s(res.hashable)); +} + +const dvo = { zip215: true }; +const _verify = (sig, msg, pub, opts = dvo) => { + msg = toU8(msg); // Message hex str/Bytes + sig = toU8(sig, 64); // Signature hex str/Bytes, must be 64 bytes + const { zip215 } = opts; // switch between zip215 and rfc8032 verif + let A, R, s, SB, hashable = new Uint8Array(); + try { + A = Point.fromHex(pub, zip215); // public key A decoded + R = Point.fromHex(sig.slice(0, 32), zip215); // 0 <= R < 2^256: ZIP215 R can be >= P + s = b2n_LE(sig.slice(32, 64)); // Decode second half as an integer S + SB = G.mul(s, false); // in the range 0 <= s < L + hashable = concatB(R.toRawBytes(), A.toRawBytes(), msg); // dom2(F, C) || R || A || PH(M) + } + catch (error) { } + const finish = (hashed) => { + if (SB == null) + return false; // false if try-catch catched an error + if (!zip215 && A.isSmallOrder()) + return false; // false for SBS: Strongly Binding Signature + const k = modL_LE(hashed); // decode in little-endian, modulo L + const RkA = R.add(A.mul(k, false)); // [8]R + [8][k]A' + return RkA.add(SB.negate()).clearCofactor().is0(); // [8][S]B = [8]R + [8][k]A' + }; + return { hashable, finish }; +}; +const verifyAsync = async (s, m, p, opts = dvo) => hashFinish(true, _verify(s, m, p, opts)); +const cr = () => // We support: 1) browsers 2) node.js 19+ + typeof globalThis === 'object' && 'crypto' in globalThis ? globalThis.crypto : undefined; +const etc = { + mod, invert, + sha512Async: async (...messages) => { + const crypto = cr(); + if (!crypto || !crypto.subtle) + err('crypto.subtle or etc.sha512Async must be defined'); + const m = concatB(...messages); + return u8n(await crypto.subtle.digest('SHA-512', m.buffer)); + }, +}; +Object.defineProperties(etc, { sha512Sync: { + configurable: false, get() { return _shaS; }, set(f) { if (!_shaS) + _shaS = f; }, + } }); +const W = 8; // Precomputes-related code. W = window size +const precompute = () => { + const points = []; // 10x sign(), 2x verify(). To achieve this, + const windows = 256 / W + 1; // app needs to spend 40ms+ to calculate + let p = G, b = p; // a lot of points related to base point G. + for (let w = 0; w < windows; w++) { // Points are stored in array and used + b = p; // any time Gx multiplication is done. + points.push(b); // They consume 16-32 MiB of RAM. + for (let i = 1; i < 2 ** (W - 1); i++) { + b = b.add(p); + points.push(b); + } + p = b.double(); // Precomputes don't speed-up getSharedKey, + } // which multiplies user point by scalar, + return points; // when precomputes are using base point +}; +let Gpows = undefined; // precomputes for base point G +const wNAF = (n) => { + // Compared to other point mult methods, + const comp = Gpows || (Gpows = precompute()); // stores 2x less points using subtraction + const neg = (cnd, p) => { let n = p.negate(); return cnd ? n : p; }; // negate + let p = I, f = G; // f must be G, or could become I in the end + const windows = 1 + 256 / W; // W=8 17 windows + const wsize = 2 ** (W - 1); // W=8 128 window size + const mask = BigInt(2 ** W - 1); // W=8 will create mask 0b11111111 + const maxNum = 2 ** W; // W=8 256 + const shiftBy = BigInt(W); // W=8 8 + for (let w = 0; w < windows; w++) { + const off = w * wsize; + let wbits = Number(n & mask); // extract W bits. + n >>= shiftBy; // shift number by W bits. + if (wbits > wsize) { + wbits -= maxNum; + n += 1n; + } // split if bits > max: +224 => 256-32 + const off1 = off, off2 = off + Math.abs(wbits) - 1; // offsets, evaluate both + const cnd1 = w % 2 !== 0, cnd2 = wbits < 0; // conditions, evaluate both + if (wbits === 0) { + f = f.add(neg(cnd1, comp[off1])); // bits are 0: add garbage to fake point + } + else { // ^ can't add off2, off2 = I + p = p.add(neg(cnd2, comp[off2])); // bits are 1: add to result point + } + } + return { p, f }; // return both real and fake points for JIT +}; \ No newline at end of file From 4062ea1fbd4d365956bb2d5ac2d01ee58332ea9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Wed, 28 Aug 2024 11:32:58 +0300 Subject: [PATCH 06/15] chore: remove forgotten logs --- src/main/java/com/limechain/client/LightClient.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/limechain/client/LightClient.java b/src/main/java/com/limechain/client/LightClient.java index bdcf1e934..281f6bdec 100644 --- a/src/main/java/com/limechain/client/LightClient.java +++ b/src/main/java/com/limechain/client/LightClient.java @@ -33,13 +33,11 @@ public LightClient() { @SneakyThrows public void start() { this.network.start(); - System.out.println("Network started"); while (true) { this.network.updateCurrentSelectedPeer(); if (this.network.getKademliaService().getSuccessfulBootNodes() > 0) { log.log(Level.INFO, "Node successfully connected to a peer! Sync can start!"); this.warpSyncMachine = AppBean.getBean(WarpSyncMachine.class); - System.out.println("Starting warp sync machine..."); this.warpSyncMachine.start(); break; } From 361d11674b2997b98642071d60eb91f12cfd0e05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Wed, 28 Aug 2024 12:05:36 +0300 Subject: [PATCH 07/15] chore: remove old logs --- .../java/com/limechain/network/StrictProtocolBinding.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/java/com/limechain/network/StrictProtocolBinding.java b/src/main/java/com/limechain/network/StrictProtocolBinding.java index a33898dc9..9e7675b84 100644 --- a/src/main/java/com/limechain/network/StrictProtocolBinding.java +++ b/src/main/java/com/limechain/network/StrictProtocolBinding.java @@ -15,7 +15,6 @@ protected StrictProtocolBinding(String protocolId/*, T protocol*/) { public Stream dialPeer(/*PeerId peer*/) { Object peer1 = getPeer(); - System.out.println("Peer: " + peer1); JSPromise dial = dial(peer1, protocolId); final var lock = new Object(); AtomicReference stream = new AtomicReference<>(); @@ -35,15 +34,10 @@ public Stream dialPeer(/*PeerId peer*/) { throw new RuntimeException(e); } } - Stream atomicStream = stream.get(); - System.out.println("Stream: " + toJson(atomicStream)); - return atomicStream; + return stream.get(); } - @JSBody(params = {"object"}, script = "return JSON.stringify(object);") - private static native String toJson(Object object); - @JSBody(params = {"peerId", "protocolId"}, script = "return (async () => ItPbStream.pbStream(await libp.dialProtocol(peerId, protocolId)))()") private static native JSPromise dial(Object peerId, String protocolId); From 9203a78918dcd5dfb317a5289561934b79f367eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Wed, 28 Aug 2024 12:08:29 +0300 Subject: [PATCH 08/15] chore: revert hasNext --- src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java b/src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java index 6348a2310..7ab6c8d3c 100644 --- a/src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java +++ b/src/main/java/com/limechain/polkaj/reader/ScaleCodecReader.java @@ -29,7 +29,7 @@ public ScaleCodecReader(byte[] source) { * @return true if has more elements */ public boolean hasNext() { - return true;//pos < source.length; + return pos < source.length; } /** From 5918e0fae362412fcfa5303c7cd7987ebe4ddc1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Wed, 28 Aug 2024 12:12:53 +0300 Subject: [PATCH 09/15] chore: remove stream object --- .../network/StrictProtocolBinding.java | 36 ------------------- .../network/protocol/warp/WarpSync.java | 11 +----- .../com/limechain/network/wrapper/Stream.java | 29 --------------- 3 files changed, 1 insertion(+), 75 deletions(-) delete mode 100644 src/main/java/com/limechain/network/wrapper/Stream.java diff --git a/src/main/java/com/limechain/network/StrictProtocolBinding.java b/src/main/java/com/limechain/network/StrictProtocolBinding.java index 9e7675b84..426ff72ab 100644 --- a/src/main/java/com/limechain/network/StrictProtocolBinding.java +++ b/src/main/java/com/limechain/network/StrictProtocolBinding.java @@ -1,11 +1,5 @@ package com.limechain.network; -import com.limechain.network.wrapper.Stream; -import org.teavm.jso.JSBody; -import org.teavm.jso.core.JSPromise; - -import java.util.concurrent.atomic.AtomicReference; - public abstract class StrictProtocolBinding { String protocolId; @@ -13,34 +7,4 @@ protected StrictProtocolBinding(String protocolId/*, T protocol*/) { this.protocolId = protocolId; } - public Stream dialPeer(/*PeerId peer*/) { - Object peer1 = getPeer(); - JSPromise dial = dial(peer1, protocolId); - final var lock = new Object(); - AtomicReference stream = new AtomicReference<>(); - - dial.then((result) -> { - stream.set((Stream) result); - synchronized (lock) { - lock.notify(); - } - return null; - }); - - synchronized (lock) { - try { - lock.wait(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - return stream.get(); - } - - @JSBody(params = {"peerId", "protocolId"}, script = "return (async () => ItPbStream.pbStream(await libp.dialProtocol(peerId, protocolId)))()") - private static native JSPromise dial(Object peerId, String protocolId); - - @JSBody(script = "return libp.getConnections()[0].remotePeer;") - private static native Object getPeer(); } diff --git a/src/main/java/com/limechain/network/protocol/warp/WarpSync.java b/src/main/java/com/limechain/network/protocol/warp/WarpSync.java index 71940361a..b33b363ad 100644 --- a/src/main/java/com/limechain/network/protocol/warp/WarpSync.java +++ b/src/main/java/com/limechain/network/protocol/warp/WarpSync.java @@ -1,24 +1,15 @@ package com.limechain.network.protocol.warp; -import com.limechain.exception.global.ExecutionFailedException; -import com.limechain.exception.global.ThreadInterruptedException; import com.limechain.network.StrictProtocolBinding; -import com.limechain.network.kad.dto.Host; -import com.limechain.network.kad.dto.PeerId; -import com.limechain.network.protocol.warp.dto.WarpSyncRequest; import com.limechain.network.protocol.warp.dto.WarpSyncResponse; -import com.limechain.network.wrapper.Stream; import lombok.extern.java.Log; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.logging.Level; @Log public class WarpSync extends StrictProtocolBinding { - private String protocolId; + private final String protocolId; public WarpSync(String protocolId) { super(protocolId); diff --git a/src/main/java/com/limechain/network/wrapper/Stream.java b/src/main/java/com/limechain/network/wrapper/Stream.java deleted file mode 100644 index 590049edd..000000000 --- a/src/main/java/com/limechain/network/wrapper/Stream.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.limechain.network.wrapper; - -import org.teavm.jso.JSObject; - -public class Stream implements JSObject { - public void write() { - - } - - public void writeLP(byte[] data) { - - } - - public void writePB() { - - } - - public void read() { - - } - - public Object readLP() { - return null; - } - - public void readPB() { - - } -} From 8e200af44e30ffdac0ad81b4f259c8795da8038d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Thu, 29 Aug 2024 09:04:40 +0300 Subject: [PATCH 10/15] chore: remove logs, and uncomment the new thread for warp sync --- .../limechain/sync/warpsync/WarpSyncMachine.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java b/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java index c281bf243..91f26cea9 100644 --- a/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java +++ b/src/main/java/com/limechain/sync/warpsync/WarpSyncMachine.java @@ -69,8 +69,6 @@ public void start() { .compareTo(initState.getFinalizedBlockHeader().getBlockNumber()) < 0) { this.syncState.setLightSyncState(initState); } - System.out.println(this.syncState.getLastFinalizedBlockHash()); - System.out.println(this.syncState.getLastFinalizedBlockNumber()); final Hash256 initStateHash = this.syncState.getLastFinalizedBlockHash(); @@ -79,14 +77,14 @@ public void start() { this.networkService.updateCurrentSelectedPeerWithNextBootnode(); this.warpSyncAction = new RequestFragmentsAction(initStateHash); -// new Thread(() -> { - while (this.warpSyncAction.getClass() != FinishedAction.class) { - this.handleState(); - this.nextState(); - } + new Thread(() -> { + while (this.warpSyncAction.getClass() != FinishedAction.class) { + this.handleState(); + this.nextState(); + } - finishWarpSync(); -// }).start(); + finishWarpSync(); + }).start(); } public void stop() { From 8076d9a03ccb30d724a7c27e7d85ab926d091d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Thu, 29 Aug 2024 09:05:20 +0300 Subject: [PATCH 11/15] chore: change request fragments action logs --- .../sync/warpsync/action/RequestFragmentsAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java b/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java index f5df404e3..857dd0987 100644 --- a/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java +++ b/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java @@ -56,9 +56,9 @@ public void handle(WarpSyncMachine sync) { throw new MissingObjectException("No response received."); } - log.log(Level.INFO, "Successfully received fragments from peer"); + log.log(Level.INFO, "Successfully received response from peer"); if (resp.getFragments().length == 0) { - log.log(Level.WARNING, "No fragments received."); + log.log(Level.WARNING, "The response contained no fragments - the peer is up to date."); return; } warpSyncState.setWarpSyncFragmentsFinished(resp.isFinished()); From a915a73762bd1a1a0a4471ead2a2a6c9c055707a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Thu, 29 Aug 2024 09:09:55 +0300 Subject: [PATCH 12/15] docs: add javadoc for LigtSyncState --- .../chain/lightsyncstate/LightSyncState.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java b/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java index a5025658e..3bb052cfa 100644 --- a/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java +++ b/src/main/java/com/limechain/chain/lightsyncstate/LightSyncState.java @@ -12,6 +12,12 @@ import java.util.Arrays; import java.util.Map; +/** + * Represents the light synchronization state in the genesis block. + * + *

This class contains the finalized block header, epoch changes, and + * the GRANDPA authority set necessary for light clients to synchronize with the blockchain.

+ */ @Getter @ToString public class LightSyncState { @@ -19,6 +25,13 @@ public class LightSyncState { private EpochChanges epochChanges; private AuthoritySet grandpaAuthoritySet; + /** + * Decodes LightSyncState from a map of hex-encoded strings. + * + * @param lightSyncStateMap A map with keys: "finalizedBlockHeader", "babeEpochChanges", and "grandpaAuthoritySet". + * @return A decoded LightSyncState instance. + * @throws IllegalStateException if any required data is missing. + */ public static LightSyncState decode(Map lightSyncStateMap) { String header = lightSyncStateMap.get("finalizedBlockHeader"); String epochChanges = lightSyncStateMap.get("babeEpochChanges"); From dd26c5cf7583eae43ccd2101b1c625d9fab09ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Thu, 29 Aug 2024 09:34:28 +0300 Subject: [PATCH 13/15] chore: put request in the try --- .../limechain/sync/warpsync/action/RequestFragmentsAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java b/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java index 857dd0987..af6820249 100644 --- a/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java +++ b/src/main/java/com/limechain/sync/warpsync/action/RequestFragmentsAction.java @@ -50,8 +50,8 @@ public void next(WarpSyncMachine sync) { @Override public void handle(WarpSyncMachine sync) { - WarpSyncResponse resp = sync.getNetworkService().makeWarpSyncRequest(blockHash.toString()); try { + WarpSyncResponse resp = sync.getNetworkService().makeWarpSyncRequest(blockHash.toString()); if (resp == null) { throw new MissingObjectException("No response received."); } From e128fda78fc64199e9a4cbf4217ef2e840aa7906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Thu, 29 Aug 2024 09:49:26 +0300 Subject: [PATCH 14/15] chore: remove annoying log --- src/main/java/com/limechain/utils/json/ObjectMapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/limechain/utils/json/ObjectMapper.java b/src/main/java/com/limechain/utils/json/ObjectMapper.java index 175d09ce5..9dcac6e85 100644 --- a/src/main/java/com/limechain/utils/json/ObjectMapper.java +++ b/src/main/java/com/limechain/utils/json/ObjectMapper.java @@ -52,7 +52,6 @@ private Field findField(Class clazz, String fieldName) { if (failOnUnknownField) { throw new IllegalStateException("Field " + fieldName + " does not exist in " + clazz.getName()); } else { - log.log(Level.FINE, "Field " + fieldName + " does not exist in " + clazz.getName()); return null; } } From 238925b854d1e59979487b41c21abe85053be5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9Cur=D0=B0d=20H=D0=B0mz=D0=B0?= Date: Thu, 29 Aug 2024 09:56:05 +0300 Subject: [PATCH 15/15] chore: remove unused imports --- src/main/webapp/index.html | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index 517da320f..1cb2bad07 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -11,17 +11,13 @@ - - - - - +