From 0b2ac15c9669b11bb9561acc509f0a2aa1ffa758 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 13 Sep 2024 00:44:09 +0800 Subject: [PATCH] optimize compile --- cpp/CMakeLists.txt | 9 ++++- cpp/cmake/Dependencies.cmake | 10 +++--- cpp/cmake/Options.cmake | 8 +++++ cpp/ppc-framework/Helper.h | 4 +-- cpp/ppc-framework/front/IFront.h | 23 ++++++++++-- cpp/ppc-framework/protocol/Message.h | 33 +++++++++++++++++ cpp/ppc-framework/protocol/MessagePayload.h | 6 ++++ cpp/vcpkg.json | 16 +++++++++ cpp/wedpr-transport/ppc-front/CMakeLists.txt | 2 +- .../ppc-front/ppc-front/PPCChannel.h | 2 -- .../ppc-front/ppc-front/PPCChannelManager.h | 2 -- .../sdk-wrapper/java/CMakeLists.txt | 2 +- .../java/swig/wedpr_java_transport.i | 36 +++++++++++++++++++ 13 files changed, 137 insertions(+), 16 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c7c0d094..45d13348 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -72,6 +72,12 @@ set(CEM_SOURCE "") if(BUILD_CEM) set(CEM_SOURCE "wedpr-computing/ppc-cem") endif() + +set(TRANSPORT_SDK_SOURCE_LIST + wedpr-protocol + wedpr-transport/ppc-front + wedpr-transport/sdk wedpr-transport/sdk-wrapper) + set(ALL_SOURCE_LIST ${SDK_SOURCE_LIST} wedpr-crypto/ppc-crypto @@ -92,7 +98,6 @@ if(BUILD_WEDPR_TOOLKIT) include(swig) message(STATUS "Getting SWIG for Windows: ...DONE") endif() - add_subdirectory(wedpr-transport/sdk-wrapper) endif() if(BUILD_ALL) @@ -101,6 +106,8 @@ elseif(BUILD_UDF) add_sources("${UDF_SOURCE_LIST}") elseif(BUILD_SDK) add_sources("${SDK_SOURCE_LIST}") +elseif(BUILD_WEDPR_TOOLKIT) + add_sources("${TRANSPORT_SDK_SOURCE_LIST}") endif() ########### set the sources end ########### diff --git a/cpp/cmake/Dependencies.cmake b/cpp/cmake/Dependencies.cmake index 4a2183ca..d8e1bf62 100644 --- a/cpp/cmake/Dependencies.cmake +++ b/cpp/cmake/Dependencies.cmake @@ -21,17 +21,19 @@ find_package(unofficial-sodium CONFIG REQUIRED) ##### the full-dependencies ##### -if(BUILD_ALL) - find_package(TBB REQUIRED) +if(BUILD_ALL OR BUILD_WEDPR_TOOLKIT) find_package(jsoncpp REQUIRED) - + find_package(TBB REQUIRED) + find_package(gRPC REQUIRED) find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED) +endif() + +if(BUILD_ALL) # tcmalloc include(ProjectTCMalloc) find_package(SEAL REQUIRED) find_package(Kuku REQUIRED) - find_package(gRPC REQUIRED) # APSI: Note: APSI depends on seal 4.0 and Kuku 2.1 include(ProjectAPSI) diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index 9b092e36..ab319fb3 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -130,10 +130,18 @@ macro(configure_project) set(VISIBILITY_FLAG "") set(BUILD_ALL OFF) endif() + if (BUILD_WEDPR_TOOLKIT) + set(VISIBILITY_FLAG "") + set(BUILD_ALL OFF) + endif() if (BUILD_ALL) # install all dependencies list(APPEND VCPKG_MANIFEST_FEATURES "all") endif() + if (BUILD_WEDPR_TOOLKIT) + # install wedpr dependencies + list(APPEND VCPKG_MANIFEST_FEATURES "toolkit") + endif() if(ENABLE_SSE) # enable sse for libhdfs3 list(APPEND VCPKG_MANIFEST_FEATURES "sse") diff --git a/cpp/ppc-framework/Helper.h b/cpp/ppc-framework/Helper.h index 13f72c46..d084cff7 100644 --- a/cpp/ppc-framework/Helper.h +++ b/cpp/ppc-framework/Helper.h @@ -18,8 +18,8 @@ * @date 2022-10-20 */ #pragma once -#include #include +#include namespace ppc { constexpr static int MAX_PORT = 65535; @@ -42,7 +42,7 @@ inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept template inline std::string_view printNodeID(T const& nodeID) { - size_t offset = nodeID.size() >= 8 ? 8 : nodeID.size(); + size_t offset = nodeID.size() >= 15 ? 15 : nodeID.size(); return std::string_view((const char*)nodeID.data(), offset); } } // namespace ppc \ No newline at end of file diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index a2e9a83d..1c0c49ea 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -162,11 +162,17 @@ class IFront : virtual public IFrontClient ppc::protocol::MessageCallback callback) = 0; /////// to simplify SDK wrapper //// + + // !!! Note: the 'payload' type(char*) should not been changed, since it used to pass-in java + // byte[] data virtual void asyncSendMessage(uint16_t routeType, - ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, - long timeout, ErrorCallback::Ptr errorCallback, IMessageHandler::Ptr msgHandler) + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload, + uint64_t payloadSize, int seq, long timeout, ErrorCallback::Ptr errorCallback, + IMessageHandler::Ptr msgHandler) { - asyncSendMessage(routeType, routeInfo, std::move(payload), seq, timeout, + // TODO: optimize here + bcos::bytes copyedPayload(payload, payload + payloadSize); + asyncSendMessage(routeType, routeInfo, std::move(copyedPayload), seq, timeout, populateErrorCallback(errorCallback), populateMsgCallback(msgHandler)); } @@ -174,6 +180,9 @@ class IFront : virtual public IFrontClient bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) = 0; /////// to simplify SDK wrapper //// + + // !!! Note: the 'payload ' type(char*) should not been changed, since it used to pass-in java + // byte[] data virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, bcos::bytes&& payload, int seq, ErrorCallback::Ptr errorCallback) { @@ -186,6 +195,14 @@ class IFront : virtual public IFrontClient ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, long timeout) = 0; + // TODO: optmize here + virtual bcos::Error::Ptr push(uint16_t routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload, + uint64_t payloadSize, int seq, long timeout) + { + bcos::bytes copyedPayload(payload, payload + payloadSize); + return push(routeType, routeInfo, std::move(copyedPayload), seq, timeout); + } virtual ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) = 0; virtual ppc::protocol::Message::Ptr peek(std::string const& topic) = 0; diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index 8f490643..2988fc4b 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -21,6 +21,7 @@ #include "MessagePayload.h" #include "RouteType.h" #include "ppc-framework/Helper.h" +#include "ppc-framework/libwrapper/Buffer.h" #include #include #include @@ -46,12 +47,34 @@ class MessageOptionalHeader // the source nodeID that send the message virtual bcos::bytes const& srcNode() const { return m_srcNode; } + /// for swig-wrapper(pass the binary data) + OutputBuffer srcNodeBuffer() const + { + // Note: this will be copied to java through jni + return OutputBuffer{(unsigned char*)m_srcNode.data(), m_srcNode.size()}; + } + virtual void setSrcNode(bcos::bytes const& srcNode) { m_srcNode = srcNode; } + // !!! Note: the first paramater type should not been changed, for it's used for pass-in java + // byte[] into c bytes + virtual void setSrcNode(char* data, uint64_t length) { m_srcNode.assign(data, data + length); } + // the target nodeID that should receive the message virtual bcos::bytes const& dstNode() const { return m_dstNode; } + + // for swig-wrapper(pass the binary to java) + OutputBuffer dstNodeBuffer() const + { + // Note: this will be copied to java through jni + return OutputBuffer{(unsigned char*)m_dstNode.data(), m_dstNode.size()}; + } virtual void setDstNode(bcos::bytes const& dstNode) { m_dstNode = dstNode; } + // !!! Note: the first paramater type(char*) should not been changed, for it's used for pass-in + // java byte[] into c bytes + virtual void setDstNode(char* data, uint64_t length) { m_dstNode.assign(data, data + length); } + // the target agency that need receive the message virtual std::string const& dstInst() const { return m_dstInst; } virtual void setDstInst(std::string const& dstInst) { m_dstInst = dstInst; } @@ -184,6 +207,16 @@ class Message : virtual public bcos::boostssl::MessageFace } std::shared_ptr payload() const override { return m_payload; } + // for swig wrapper + OutputBuffer payloadBuffer() const + { + if (!m_payload) + { + return OutputBuffer{nullptr, 0}; + } + return OutputBuffer{(unsigned char*)m_payload->data(), m_payload->size()}; + } + void setPayload(std::shared_ptr _payload) override { m_payload = std::move(_payload); diff --git a/cpp/ppc-framework/protocol/MessagePayload.h b/cpp/ppc-framework/protocol/MessagePayload.h index f5fbf9e3..a8e6a57c 100644 --- a/cpp/ppc-framework/protocol/MessagePayload.h +++ b/cpp/ppc-framework/protocol/MessagePayload.h @@ -18,6 +18,7 @@ * @date 2024-08-22 */ #pragma once +#include "ppc-framework/libwrapper/Buffer.h" #include #include @@ -42,6 +43,11 @@ class MessagePayload virtual void setVersion(uint8_t version) { m_version = version; } // data virtual bcos::bytes const& data() const { return m_data; } + // for swig wrapper here + virtual OutputBuffer dataBuffer() const + { + return OutputBuffer{(unsigned char*)m_data.data(), m_data.size()}; + } virtual void setData(bcos::bytes&& data) { m_data = std::move(data); } virtual void setData(bcos::bytes const& data) { m_data = data; } // the seq diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index a56cfe04..83449a54 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -103,6 +103,22 @@ "tbb", "libxml2" ] + }, + "toolkit": { + "description": "tooklit dependencies", + "dependencies": [ + { + "name": "bcos-boostssl", + "version>=": "3.2.3" + }, + { + "name": "grpc", + "version>=": "1.51.1" + }, + "tarscpp", + "tbb", + "libxml2" + ] } } } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/CMakeLists.txt b/cpp/wedpr-transport/ppc-front/CMakeLists.txt index efd94094..8966cbae 100644 --- a/cpp/wedpr-transport/ppc-front/CMakeLists.txt +++ b/cpp/wedpr-transport/ppc-front/CMakeLists.txt @@ -4,7 +4,7 @@ aux_source_directory(ppc-front SRCS) add_library(${FRONT_TARGET} ${SRCS}) -target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc) +target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TBB::tbb) #if (TESTS) # enable_testing() diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h index f881a915..3f5e4edf 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h @@ -23,7 +23,6 @@ #include "Common.h" #include "Front.h" #include "ppc-framework/front/Channel.h" -#include #include namespace ppc::front @@ -43,7 +42,6 @@ class PPCChannel : public Channel, public std::enable_shared_from_this().swap(m_handlers); std::unordered_map().swap(m_messages); - MallocExtension::instance()->ReleaseFreeMemory(); FRONT_LOG(INFO) << LOG_DESC("the PPCChannel destroyed"); } diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h index 3e4b6bd4..506b2416 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h @@ -21,7 +21,6 @@ #pragma once #include "PPCChannel.h" -#include #include namespace ppc::front @@ -37,7 +36,6 @@ class PPCChannelManager : public std::enable_shared_from_this { std::unordered_map().swap(m_holdingMessages); std::unordered_map().swap(m_channels); - MallocExtension::instance()->ReleaseFreeMemory(); FRONT_LOG(INFO) << LOG_DESC("the PPCChannelManager destroyed"); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt index 806aebfd..9276c9e6 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt @@ -1,4 +1,4 @@ -file(GLOB_RECURSE SRCS *.i) +file(GLOB_RECURSE SRCS wedpr_java_transport.i) set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON) set(WEDPR_TRANSPORT_PACKAGE "com.webank.wedpr.sdk.jni.generated") diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i index 8f64b07f..35dfdd9b 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i @@ -31,6 +31,7 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long); %include %include %include +%include // shared_ptr definition %shared_ptr(ppc::front::FrontConfig); @@ -59,9 +60,11 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long); %{ #define SWIG_FILE_WITH_INIT #include +#include #include #include "wedpr-transport/sdk/TransportBuilder.h" #include "wedpr-transport/sdk/Transport.h" +#include "ppc-framework/libwrapper/Buffer.h" #include "ppc-framework/front/IFront.h" #include "ppc-framework/protocol/RouteType.h" #include "ppc-framework/front/FrontConfig.h" @@ -151,8 +154,41 @@ namespace bcos{ %feature("director") ppc::front::MessageDispatcherHandler; %feature("director") ppc::front::IMessageHandler; +// Note: the field data should equal to the fieldMap of class or the function +%include various.i +// this means convert all (char*) to byte[] +%apply char *BYTE {char * }; + + +%typemap(jni) OutputBuffer "jbyteArray" +%typemap(jtype) OutputBuffer "byte[]" +%typemap(jstype) OutputBuffer "byte[]" +%typemap(in) OutputBuffer { + $1.data = (unsigned char *) JCALL2(GetByteArrayElements, jenv, $input, 0); + $1.len = JCALL1(GetArrayLength, jenv, $input); +} +%typemap(argout) OutputBuffer { + JCALL3(ReleaseByteArrayElements, jenv, $input, (jbyte *) $1.data, 0); +} +// Note: will cause copy herer +%typemap(out) OutputBuffer { + $result = JCALL1(NewByteArray, jenv, $1.len); + JCALL4(SetByteArrayRegion, jenv, $result, 0, $1.len, (jbyte *) $1.data); + delete[] $1.data; +} +%typemap(javain) OutputBuffer "$javainput" +%typemap(javaout) OutputBuffer { return $jnicall; } + +/* +///// tests /// +%inline { +} +///// tests /// +*/ + // define the interface should been exposed %include "bcos-utilities/Error.h" +%include "ppc-framework/libwrapper/Buffer.h" %include "ppc-framework/front/FrontConfig.h" %include "ppc-framework/protocol/EndPoint.h" %include "ppc-framework/protocol/GrpcConfig.h"