complement config && optmize compile

WeBankBlockchain · Sep 5, 2024 · 0822ed5 · 0822ed5
1 parent cddb3ee
commit 0822ed5
Show file tree

Hide file tree

Showing 71 changed files with 794 additions and 460 deletions.
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -72,14 +72,6 @@ set(CEM_SOURCE "")
 if(BUILD_CEM)
     set(CEM_SOURCE "wedpr-computing/ppc-cem")
 endif()
-#set(ALL_SOURCE_LIST 
-#        ${SDK_SOURCE_LIST} ppc-crypto 
-#        libhelper libinitializer ppc-io ppc-protocol 
-#        ppc-gateway ppc-front  ppc-tars-protocol 
-#        wedpr-helper/ppc-tools ppc-storage ppc-psi ppc-rpc 
-#        ppc-http ppc-mpc ppc-pir
-#        ${CEM_SOURCE} ppc-main)
-
 set(ALL_SOURCE_LIST 
         ${SDK_SOURCE_LIST} 
         wedpr-crypto/ppc-crypto 

diff --git a/cpp/ppc-framework/front/FrontConfig.h b/cpp/ppc-framework/front/FrontConfig.h
@@ -34,43 +34,32 @@ class FrontConfig
 {
 public:
     using Ptr = std::shared_ptr<FrontConfig>;
+    FrontConfig() = default;
+
     FrontConfig(int threadPoolSize, std::string nodeID)
       : m_threadPoolSize(threadPoolSize), m_nodeID(std::move(nodeID))
     {}
     virtual ~FrontConfig() = default;
 
     virtual int threadPoolSize() const { return m_threadPoolSize; }
-    virtual std::string const& nodeID() const { return m_nodeID; }
-    virtual std::vector<ppc::protocol::EndPoint> const& gatewayInfo() const
-    {
-        return m_gatewayInfo;
-    }
-    virtual void setGatewayInfo(std::vector<ppc::protocol::EndPoint> gatewayInfo)
-    {
-        m_gatewayInfo = std::move(gatewayInfo);
-    }
+    virtual void setThreadPoolSize(int threadPoolSize) { m_threadPoolSize = threadPoolSize; }
 
-    virtual void appendGatewayInfo(ppc::protocol::EndPoint&& endpoint)
-    {
-        m_gatewayInfo.push_back(endpoint);
-    }
+    virtual std::string const& nodeID() const { return m_nodeID; }
+    virtual void setNodeID(std::string const& nodeID) { m_nodeID = nodeID; }
 
     ppc::protocol::EndPoint const& selfEndPoint() const { return m_selfEndPoint; }
+    ppc::protocol::EndPoint& mutableSelfEndPoint() { return m_selfEndPoint; }
+
     void setSelfEndPoint(ppc::protocol::EndPoint const& endPoint) { m_selfEndPoint = endPoint; }
 
+    void setGatewayGrpcTarget(std::string const& gatewayGrpcTarget)
+    {
+        m_gatewayGrpcTarget = gatewayGrpcTarget;
+    }
     // refer to: https://github.com/grpc/grpc-node/issues/2066
     // grpc prefer to using ipv4:${host1}:${port1},${host2}:${port2} as target to support multiple
     // servers
-    std::string gatewayGrpcTarget()
-    {
-        std::stringstream oss;
-        oss << "ipv4:";
-        for (auto const& endPoint : m_gatewayInfo)
-        {
-            oss << endPoint.entryPoint() << ",";
-        }
-        return oss.str();
-    }
+    std::string const& gatewayGrpcTarget() const { return m_gatewayGrpcTarget; }
 
     void setGrpcConfig(ppc::protocol::GrpcConfig::Ptr grpcConfig)
     {
@@ -84,15 +73,28 @@ class FrontConfig
     virtual std::vector<std::string> const& getComponents() const { return m_components; }
     void setComponents(std::vector<std::string> const& components) { m_components = components; }
 
+    std::vector<std::string>& mutableComponents() { return m_components; }
+
 protected:
     ppc::protocol::GrpcConfig::Ptr m_grpcConfig;
     ppc::protocol::EndPoint m_selfEndPoint;
     int m_threadPoolSize;
     std::string m_nodeID;
-    std::vector<ppc::protocol::EndPoint> m_gatewayInfo;
+    std::string m_gatewayGrpcTarget;
     std::vector<std::string> m_components;
 };
 
+class FrontConfigBuilder
+{
+public:
+    using Ptr = std::shared_ptr<FrontConfigBuilder>;
+    FrontConfigBuilder() = default;
+    virtual ~FrontConfigBuilder() = default;
+
+    virtual FrontConfig::Ptr build() const = 0;
+    virtual FrontConfig::Ptr build(int threadPoolSize, std::string nodeID) const = 0;
+};
+
 inline std::string printFrontDesc(FrontConfig::Ptr const& config)
 {
     if (!config)

diff --git a/cpp/ppc-framework/protocol/EndPoint.h b/cpp/ppc-framework/protocol/EndPoint.h
@@ -41,6 +41,7 @@ class EndPoint
 
     void setHost(std::string host) { m_host = std::move(host); }
     void setPort(uint16_t port) { m_port = port; }
+    void setListenIp(std::string const& listenIp) { m_listenIp = listenIp; }
 
     std::string entryPoint() const { return m_host + ":" + std::to_string(m_port); }
 

diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h
@@ -37,13 +37,13 @@ class GrpcConfig
     GrpcConfig() = default;
     virtual ~GrpcConfig() = default;
 
-    std::string const& loadBalancePolicy() const { return m_loadBanlancePolicy; }
-    void setLoadBalancePolicy(std::string const& loadBanlancePolicy)
+    std::string const& loadBalancePolicy() const { return m_loadBalancePolicy; }
+    void setLoadBalancePolicy(std::string const& loadBalancePolicy)
     {
-        m_loadBanlancePolicy = loadBanlancePolicy;
+        m_loadBalancePolicy = loadBalancePolicy;
     }
 
 private:
-    std::string m_loadBanlancePolicy = "round_robin";
+    std::string m_loadBalancePolicy = "round_robin";
 };
 }  // namespace ppc::protocol
diff --git a/cpp/ppc-framework/psi/TaskFrameworkInterface.h b/cpp/ppc-framework/psi/TaskFrameworkInterface.h
diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh
@@ -10,9 +10,10 @@ listen_ip="0.0.0.0"
 cdn_link_header="https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/FISCO-BCOS"
 OPENSSL_CMD="${HOME}/.fisco/tassl-1.1.1b"
 
-port_start=(40300 10200)
+port_start=(40300 10200 18000)
 gateway_listen_port=port_start[0]
 rpc_listen_port=port_start[1]
+grpc_listen_port=port_start[2]
 
 use_ip_param=
 ip_param=
@@ -357,16 +358,17 @@ generate_config_ini() {
 
     local rpc_listen_ip="${4}"
     local rpc_listen_port="${5}"
-    local agency_info="${6}"
-    local agency_id="${7}"
-    local index="${8}"
+
+    local grpc_listen_ip="${6}"
+    local grpc_listen_port="${7}"
+
+    local agency_id="${8}"
+    local index="${9}"
 
     cat <<EOF >"${output}"
 [agency]
     ; the agency-id of self-party
     id = ${agency_id}
-    ; the agency info
-    ${agency_info}
     ; the private-key for the psi-server
     private_key_path = conf/node.pem
     ; disable the ra2018 or not, default enable ra2018
@@ -376,6 +378,18 @@ generate_config_ini() {
     ; task_timeout_minutes = 180
     ; thread_count = 8
 
+[transport]
+   ; the endpoint information
+   listen_ip = ${grpc_listen_ip}
+   listen_port = ${grpc_listen_port}
+   host_ip = 
+   ; the threadPoolSize
+   thread_count = 4
+   ; the gatewayService endpoint information
+   service.gateway_target =  
+   ; the components
+   service.components =
+
 [crypto]
     sm_crypto = ${sm_mode}
 
@@ -385,14 +399,19 @@ generate_config_ini() {
     ; thread_count = 4
     ; ssl or sm ssl
     sm_ssl=${sm_mode}
-	  ; the max allowed message size in MBytes, default is 100MBytes
-	  max_allow_msg_size = 100
+    ; the max allowed message size in MBytes, default is 100MBytes
+    max_allow_msg_size = 100
     ;ssl connection switch, if disable the ssl connection, default: false
     ;disable_ssl = true
     ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes
     holding_msg_minutes = 30
-    ;disable_cache = false
     ;reconnect_time = 10000
+    ; the unreachable distance
+    ;unreachable_distance=10
+    ;the dir that contains the connected endpoint information, e.g.nodes.json 
+    ;nodes_path=./
+    ; the file that configure the connected endpoint information
+    ; nodes_path=nodes.json
 
 [rpc]
     listen_ip=${rpc_listen_ip}
@@ -411,22 +430,6 @@ generate_config_ini() {
     ; directory the certificates located in
     cert_path=./conf
 
-[cache]
-    ; the cache type, only support redis now
-    type = 0
-    proxy = 127.0.0.1:20002
-    obServer = 127.0.0.1:10904
-    cluster = REDIS_CLUSTER
-    user = 1194
-    host = 127.0.0.1
-    port = 6379
-    password = 
-    database = 1
-    pool_size = 16
-    ; the redis connection timeout, in ms, default is 500ms
-    connection_timeout = 500
-    ; the redis socket timeout, in ms, default is 500ms
-    socket_timeout = 500
 
 [storage]
     host = 127.0.0.1
@@ -813,7 +816,6 @@ deploy_nodes()
     fi
 
     local agency_index=0
-    local agency_info=""
     # generate the ca-cert
     ca_dir="${output_dir}"/ca
     generate_ca_cert "${sm_mode}" "${ca_dir}"
@@ -847,8 +849,6 @@ deploy_nodes()
             # generate the node-script
             generate_node_scripts "${node_dir}"
             local port=$((gateway_listen_port + node_count))
-            # generate the agency_info
-            agency_info="${agency_info}agency.agency${agency_index}=${ip}:${port}
     "
             ((agency_index += 1))
             set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1))
@@ -872,9 +872,10 @@ deploy_nodes()
             node_dir="${output_dir}/${ip}/node${node_count}"
             local gateway_port=$((gateway_listen_port + node_count))
             local rpc_port=$((rpc_listen_port + node_count))
+            local grpc_port=$((grpc_listen_port + node_count))
             local agency_id="agency${count}"
             private_key=$(generate_private_key "${node_dir}/conf")
-            generate_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id} "${count}"
+            generate_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}"
             set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1))
             ((++count))
         done

diff --git a/cpp/wedpr-computing/ppc-cem/src/CEMService.cpp b/cpp/wedpr-computing/ppc-cem/src/CEMService.cpp
@@ -156,7 +156,7 @@ void CEMService::makeCiphertextEqualityMatch(
         }
     });
     CEM_LOG(INFO) << LOG_DESC("ciphertext equality match request handle ok")
-                 << LOG_KV("timecost(ms)", utcSteadyTime() - startT);
+                  << LOG_KV("timecost(ms)", utcSteadyTime() - startT);
 }
 
 void CEMService::makeCiphertextEqualityMatchRpc(Json::Value const& request, RespFunc func)
@@ -266,7 +266,7 @@ void CEMService::encryptDataset(Json::Value const& request, Json::Value& respons
         }
     });
     CEM_LOG(INFO) << LOG_DESC("encrypt dataset request handle ok")
-                 << LOG_KV("timecost(ms)", utcSteadyTime() - startT);
+                  << LOG_KV("timecost(ms)", utcSteadyTime() - startT);
 }
 
 void CEMService::encryptDatasetRpc(Json::Value const& request, RespFunc func)

diff --git a/cpp/wedpr-computing/ppc-cem/src/CEMService.h b/cpp/wedpr-computing/ppc-cem/src/CEMService.h
@@ -19,19 +19,13 @@
  */
 #pragma once
 #include "Common.h"
+#include "ppc-framework/rpc/RpcInterface.h"
 #include "ppc-io/src/FileLineReader.h"
 #include "ppc-io/src/FileLineWriter.h"
-#include "ppc-rpc/src/RpcFactory.h"
-#include "ppc-tools/src/config/PPCConfig.h"
+#include "ppc-tools/src/config/CEMConfig.h"
+#include "ppc-tools/src/config/StorageConfig.h"
 #include <bcos-utilities/Common.h>
 #include <string>
-
-using namespace bcos;
-using namespace ppc;
-using namespace ppc::io;
-using namespace ppc::rpc;
-using namespace ppc::tools;
-
 namespace ppc::cem
 {
 class CEMService
@@ -41,23 +35,26 @@ class CEMService
     CEMService() = default;
     virtual ~CEMService() = default;
 
-    void makeCiphertextEqualityMatchRpc(Json::Value const& request, RespFunc func);
-    void makeCiphertextEqualityMatch(Json::Value const& request, Json::Value& response, ppc::protocol::DataResourceType _type);
-    void encryptDatasetRpc(Json::Value const& request, RespFunc func);
+    void makeCiphertextEqualityMatchRpc(Json::Value const& request, ppc::rpc::RespFunc func);
+    void makeCiphertextEqualityMatch(
+        Json::Value const& request, Json::Value& response, ppc::protocol::DataResourceType _type);
+    void encryptDatasetRpc(Json::Value const& request, ppc::rpc::RespFunc func);
     void encryptDataset(Json::Value const& request, Json::Value& response);
 
-    void setCEMConfig(CEMConfig const& cemConfig);
-    void setStorageConfig(StorageConfig const& storageConfig);
+    void setCEMConfig(ppc::tools::CEMConfig const& cemConfig);
+    void setStorageConfig(ppc::tools::StorageConfig const& storageConfig);
     void doCipherTextEqualityMatch(const Json::Value::Members& fieldNames,
-        const std::vector<std::string>& fieldValues, LineReader::Ptr lineReader,
+        const std::vector<std::string>& fieldValues, ppc::io::LineReader::Ptr lineReader,
         Json::Value& matchCount);
-    void doEncryptDataset(LineReader::Ptr lineReader, LineWriter::Ptr lineWriter);
-    LineReader::Ptr initialize_lineReader(const std::string& _datasetId, ppc::protocol::DataResourceType _type);
-    LineWriter::Ptr initialize_lineWriter(const std::string& _datasetId, ppc::protocol::DataResourceType _type);
+    void doEncryptDataset(ppc::io::LineReader::Ptr lineReader, ppc::io::LineWriter::Ptr lineWriter);
+    ppc::io::LineReader::Ptr initialize_lineReader(
+        const std::string& _datasetId, ppc::protocol::DataResourceType _type);
+    ppc::io::LineWriter::Ptr initialize_lineWriter(
+        const std::string& _datasetId, ppc::protocol::DataResourceType _type);
     void renameSource(const std::string& _datasetId, ppc::protocol::DataResourceType _type);
 
 private:
-    CEMConfig m_cemConfig;
-    StorageConfig m_storageConfig;
+    ppc::tools::CEMConfig m_cemConfig;
+    ppc::tools::StorageConfig m_storageConfig;
 };
 }  // namespace ppc::cem