optimize compile

cpp/CMakeLists.txt

@@ -72,6 +72,12 @@ set(CEM_SOURCE "")
 if(BUILD_CEM)
     set(CEM_SOURCE "wedpr-computing/ppc-cem")
 endif()
+
+set(TRANSPORT_SDK_SOURCE_LIST 
+    wedpr-protocol 
+    wedpr-transport/ppc-front 
+    wedpr-transport/sdk wedpr-transport/sdk-wrapper)
+
 set(ALL_SOURCE_LIST 
         ${SDK_SOURCE_LIST} 
         wedpr-crypto/ppc-crypto 
@@ -92,7 +98,6 @@ if(BUILD_WEDPR_TOOLKIT)
         include(swig)
         message(STATUS "Getting SWIG for Windows: ...DONE")
     endif()
-    add_subdirectory(wedpr-transport/sdk-wrapper)
 endif()
 
 if(BUILD_ALL)
@@ -101,6 +106,8 @@ elseif(BUILD_UDF)
     add_sources("${UDF_SOURCE_LIST}")
 elseif(BUILD_SDK)
     add_sources("${SDK_SOURCE_LIST}")
+elseif(BUILD_WEDPR_TOOLKIT)
+    add_sources("${TRANSPORT_SDK_SOURCE_LIST}")
 endif()
 ########### set the sources end ########### 
 

cpp/cmake/Dependencies.cmake

@@ -21,17 +21,19 @@ find_package(unofficial-sodium CONFIG REQUIRED)
 
 
 ##### the full-dependencies #####
-if(BUILD_ALL)
-    find_package(TBB REQUIRED)
+if(BUILD_ALL OR BUILD_WEDPR_TOOLKIT)
     find_package(jsoncpp REQUIRED)
-
+    find_package(TBB REQUIRED)
+    find_package(gRPC REQUIRED)
     find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED)
+endif()
+
+if(BUILD_ALL)
     # tcmalloc
     include(ProjectTCMalloc)
 
     find_package(SEAL REQUIRED)
     find_package(Kuku REQUIRED)
-    find_package(gRPC REQUIRED)
 
     # APSI: Note: APSI depends on seal 4.0 and Kuku 2.1
     include(ProjectAPSI)

cpp/cmake/Options.cmake

@@ -130,10 +130,18 @@ macro(configure_project)
         set(VISIBILITY_FLAG "")
         set(BUILD_ALL OFF)
     endif()
+    if (BUILD_WEDPR_TOOLKIT)
+        set(VISIBILITY_FLAG "")
+        set(BUILD_ALL OFF)
+    endif()
     if (BUILD_ALL)
         # install all dependencies
         list(APPEND VCPKG_MANIFEST_FEATURES "all")
     endif()
+    if (BUILD_WEDPR_TOOLKIT)
+        # install wedpr dependencies
+        list(APPEND VCPKG_MANIFEST_FEATURES "toolkit")
+    endif()
     if(ENABLE_SSE)
         # enable sse for libhdfs3
         list(APPEND VCPKG_MANIFEST_FEATURES "sse")

cpp/ppc-framework/Helper.h

@@ -18,8 +18,8 @@
  * @date 2022-10-20
  */
 #pragma once
-#include <string>
 #include <sstream>
+#include <string>
 namespace ppc
 {
 constexpr static int MAX_PORT = 65535;
@@ -42,7 +42,7 @@ inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept
 template <typename T>
 inline std::string_view printNodeID(T const& nodeID)
 {
-    size_t offset = nodeID.size() >= 8 ? 8 : nodeID.size();
+    size_t offset = nodeID.size() >= 15 ? 15 : nodeID.size();
     return std::string_view((const char*)nodeID.data(), offset);
 }
 }  // namespace ppc

cpp/ppc-framework/front/IFront.h

@@ -162,18 +162,27 @@ class IFront : virtual public IFrontClient
         ppc::protocol::MessageCallback callback) = 0;
 
     /////// to simplify SDK wrapper ////
+
+    // !!! Note: the 'payload' type(char*) should not been changed, since it used to pass-in java
+    // byte[] data
     virtual void asyncSendMessage(uint16_t routeType,
-        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq,
-        long timeout, ErrorCallback::Ptr errorCallback, IMessageHandler::Ptr msgHandler)
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload,
+        uint64_t payloadSize, int seq, long timeout, ErrorCallback::Ptr errorCallback,
+        IMessageHandler::Ptr msgHandler)
     {
-        asyncSendMessage(routeType, routeInfo, std::move(payload), seq, timeout,
+        // TODO: optimize here
+        bcos::bytes copyedPayload(payload, payload + payloadSize);
+        asyncSendMessage(routeType, routeInfo, std::move(copyedPayload), seq, timeout,
             populateErrorCallback(errorCallback), populateMsgCallback(msgHandler));
     }
 
     virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID,
         bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) = 0;
 
     /////// to simplify SDK wrapper  ////
+
+    // !!! Note: the 'payload ' type(char*) should not been changed, since it used to pass-in java
+    // byte[] data
     virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID,
         bcos::bytes&& payload, int seq, ErrorCallback::Ptr errorCallback)
     {
@@ -186,6 +195,14 @@ class IFront : virtual public IFrontClient
         ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq,
         long timeout) = 0;
 
+    // TODO: optmize here
+    virtual bcos::Error::Ptr push(uint16_t routeType,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload,
+        uint64_t payloadSize, int seq, long timeout)
+    {
+        bcos::bytes copyedPayload(payload, payload + payloadSize);
+        return push(routeType, routeInfo, std::move(copyedPayload), seq, timeout);
+    }
     virtual ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) = 0;
     virtual ppc::protocol::Message::Ptr peek(std::string const& topic) = 0;
 

cpp/ppc-framework/protocol/Message.h

@@ -21,6 +21,7 @@
 #include "MessagePayload.h"
 #include "RouteType.h"
 #include "ppc-framework/Helper.h"
+#include "ppc-framework/libwrapper/Buffer.h"
 #include <bcos-boostssl/interfaces/MessageFace.h>
 #include <bcos-utilities/Common.h>
 #include <bcos-utilities/DataConvertUtility.h>
@@ -46,12 +47,34 @@ class MessageOptionalHeader
 
     // the source nodeID that send the message
     virtual bcos::bytes const& srcNode() const { return m_srcNode; }
+    /// for swig-wrapper(pass the binary data)
+    OutputBuffer srcNodeBuffer() const
+    {
+        // Note: this will be copied to java through jni
+        return OutputBuffer{(unsigned char*)m_srcNode.data(), m_srcNode.size()};
+    }
+
     virtual void setSrcNode(bcos::bytes const& srcNode) { m_srcNode = srcNode; }
 
+    // !!! Note: the first paramater type should not been changed, for it's used for pass-in java
+    // byte[] into c bytes
+    virtual void setSrcNode(char* data, uint64_t length) { m_srcNode.assign(data, data + length); }
+
     // the target nodeID that should receive the message
     virtual bcos::bytes const& dstNode() const { return m_dstNode; }
+
+    // for swig-wrapper(pass the binary to java)
+    OutputBuffer dstNodeBuffer() const
+    {
+        // Note: this will be copied to java through jni
+        return OutputBuffer{(unsigned char*)m_dstNode.data(), m_dstNode.size()};
+    }
     virtual void setDstNode(bcos::bytes const& dstNode) { m_dstNode = dstNode; }
 
+    // !!! Note: the first paramater type(char*) should not been changed, for it's used for pass-in
+    // java byte[] into c bytes
+    virtual void setDstNode(char* data, uint64_t length) { m_dstNode.assign(data, data + length); }
+
     // the target agency that need receive the message
     virtual std::string const& dstInst() const { return m_dstInst; }
     virtual void setDstInst(std::string const& dstInst) { m_dstInst = dstInst; }
@@ -184,6 +207,16 @@ class Message : virtual public bcos::boostssl::MessageFace
     }
 
     std::shared_ptr<bcos::bytes> payload() const override { return m_payload; }
+    // for swig wrapper
+    OutputBuffer payloadBuffer() const
+    {
+        if (!m_payload)
+        {
+            return OutputBuffer{nullptr, 0};
+        }
+        return OutputBuffer{(unsigned char*)m_payload->data(), m_payload->size()};
+    }
+
     void setPayload(std::shared_ptr<bcos::bytes> _payload) override
     {
         m_payload = std::move(_payload);

cpp/ppc-framework/protocol/MessagePayload.h

@@ -18,6 +18,7 @@
  * @date 2024-08-22
  */
 #pragma once
+#include "ppc-framework/libwrapper/Buffer.h"
 #include <bcos-utilities/Common.h>
 #include <memory>
 
@@ -42,6 +43,11 @@ class MessagePayload
     virtual void setVersion(uint8_t version) { m_version = version; }
     // data
     virtual bcos::bytes const& data() const { return m_data; }
+    // for swig wrapper here
+    virtual OutputBuffer dataBuffer() const
+    {
+        return OutputBuffer{(unsigned char*)m_data.data(), m_data.size()};
+    }
     virtual void setData(bcos::bytes&& data) { m_data = std::move(data); }
     virtual void setData(bcos::bytes const& data) { m_data = data; }
     // the seq

cpp/vcpkg.json

@@ -103,6 +103,22 @@
         "tbb",
         "libxml2"
       ]
+    },
+    "toolkit": {
+      "description": "tooklit dependencies",
+      "dependencies": [
+        {
+          "name": "bcos-boostssl",
+          "version>=": "3.2.3"
+        },
+        {
+          "name": "grpc",
+          "version>=": "1.51.1"
+        },
+        "tarscpp",
+        "tbb",
+        "libxml2"
+      ]
     }
   }
 }

cpp/wedpr-transport/ppc-front/CMakeLists.txt

@@ -4,7 +4,7 @@ aux_source_directory(ppc-front SRCS)
 
 
 add_library(${FRONT_TARGET} ${SRCS})
-target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc)
+target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TBB::tbb)
 
 #if (TESTS)
 #    enable_testing()

cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h

@@ -23,7 +23,6 @@
 #include "Common.h"
 #include "Front.h"
 #include "ppc-framework/front/Channel.h"
-#include <gperftools/malloc_extension.h>
 #include <utility>
 
 namespace ppc::front
@@ -43,7 +42,6 @@ class PPCChannel : public Channel, public std::enable_shared_from_this<PPCChanne
     {
         std::unordered_map<uint64_t, MessageHandler::Ptr>().swap(m_handlers);
         std::unordered_map<uint64_t, front::PPCMessageFace::Ptr>().swap(m_messages);
-        MallocExtension::instance()->ReleaseFreeMemory();
         FRONT_LOG(INFO) << LOG_DESC("the PPCChannel destroyed");
     }
 

cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h

@@ -21,7 +21,6 @@
 #pragma once
 
 #include "PPCChannel.h"
-#include <gperftools/malloc_extension.h>
 #include <thread>
 
 namespace ppc::front
@@ -37,7 +36,6 @@ class PPCChannelManager : public std::enable_shared_from_this<PPCChannelManager>
     {
         std::unordered_map<std::string, HoldingMessage::Ptr>().swap(m_holdingMessages);
         std::unordered_map<std::string, front::Channel::Ptr>().swap(m_channels);
-        MallocExtension::instance()->ReleaseFreeMemory();
         FRONT_LOG(INFO) << LOG_DESC("the PPCChannelManager destroyed");
     }
 

cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt

@@ -1,4 +1,4 @@
-file(GLOB_RECURSE SRCS *.i)
+file(GLOB_RECURSE SRCS wedpr_java_transport.i)
 set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON)
 
 set(WEDPR_TRANSPORT_PACKAGE "com.webank.wedpr.sdk.jni.generated")

cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i

@@ -31,6 +31,7 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long);
 %include <std_vector.i>
 %include <std_string.i>
 %include <std_shared_ptr.i>
+%include <various.i>
 
 // shared_ptr definition
 %shared_ptr(ppc::front::FrontConfig);
@@ -59,9 +60,11 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long);
 %{
 #define SWIG_FILE_WITH_INIT
 #include <vector>
+#include <iostream>
 #include <stdint.h>
 #include "wedpr-transport/sdk/TransportBuilder.h"
 #include "wedpr-transport/sdk/Transport.h"
+#include "ppc-framework/libwrapper/Buffer.h"
 #include "ppc-framework/front/IFront.h"
 #include "ppc-framework/protocol/RouteType.h"
 #include "ppc-framework/front/FrontConfig.h"
@@ -151,8 +154,41 @@ namespace bcos{
 %feature("director") ppc::front::MessageDispatcherHandler;
 %feature("director") ppc::front::IMessageHandler;
 
+// Note: the field data should equal to the fieldMap of class or the function
+%include various.i 
+// this means convert all (char*) to byte[]
+%apply char *BYTE {char * };
+
+
+%typemap(jni) OutputBuffer "jbyteArray"
+%typemap(jtype) OutputBuffer "byte[]"
+%typemap(jstype) OutputBuffer "byte[]"
+%typemap(in) OutputBuffer {
+    $1.data = (unsigned char *) JCALL2(GetByteArrayElements, jenv, $input, 0);
+    $1.len = JCALL1(GetArrayLength, jenv, $input);
+}
+%typemap(argout) OutputBuffer {
+    JCALL3(ReleaseByteArrayElements, jenv, $input, (jbyte *) $1.data, 0);
+}
+// Note: will cause copy herer
+%typemap(out) OutputBuffer {
+    $result = JCALL1(NewByteArray, jenv, $1.len);
+    JCALL4(SetByteArrayRegion, jenv, $result, 0, $1.len, (jbyte *) $1.data);
+    delete[] $1.data;
+}
+%typemap(javain) OutputBuffer "$javainput"
+%typemap(javaout) OutputBuffer { return $jnicall; }
+
+/*
+///// tests  ///
+%inline {
+}
+///// tests  ///
+*/
+
 // define the interface should been exposed
 %include "bcos-utilities/Error.h"
+%include "ppc-framework/libwrapper/Buffer.h"
 %include "ppc-framework/front/FrontConfig.h"
 %include "ppc-framework/protocol/EndPoint.h"
 %include "ppc-framework/protocol/GrpcConfig.h"