Skip to content

Commit

Permalink
air node gateway not required to load transport config (#97)
Browse files Browse the repository at this point in the history
* fix pir link

* air node gateway not required to load transport config

* optimize link

* optmize link
  • Loading branch information
cyjseagull authored Nov 20, 2024
1 parent 6164668 commit acbdd04
Show file tree
Hide file tree
Showing 26 changed files with 44 additions and 35 deletions.
3 changes: 0 additions & 3 deletions cpp/ppc-framework/io/DataBatch.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
#pragma once
#include "../Common.h"
#include <bcos-utilities/Common.h>
#include <gperftools/malloc_extension.h>
#include <boost/lexical_cast.hpp>
#include <algorithm>
#include <memory>
Expand Down Expand Up @@ -210,8 +209,6 @@ class DataBatch
}
m_data->clear();
m_data.reset();
// free after release
MallocExtension::instance()->ReleaseFreeMemory();
}

private:
Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ add_library(${PIR_TARGET} ${SOURCES} ${OUT_TARS_H_LIST})
target_include_directories(${PIR_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${PIR_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc)
target_link_libraries(${PIR_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ set(TEST_BINARY_NAME test-ppc-pir)
add_executable(${TEST_BINARY_NAME} ${SOURCES})
target_include_directories(${TEST_BINARY_NAME} PRIVATE .)

target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${CRYPTO_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${PIR_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST})
target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${PIR_TARGET} ${BOOST_UNIT_TEST})
add_test(NAME test-pir WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
file(GLOB_RECURSE SRCS *.cpp)
add_library(${BS_ECDH_PSI_TARGET} ${SRCS})

target_link_libraries(${BS_ECDH_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} wedpr_ecc TBB::tbb TCMalloc)
target_link_libraries(${BS_ECDH_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} wedpr_ecc TBB::tbb)
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ add_library(${CM2020_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${CM2020_PSI_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${CM2020_PSI_TARGET} PUBLIC ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} TBB::tbb TCMalloc)
target_link_libraries(${CM2020_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} TBB::tbb)
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ add_library(${ECDH_CONN_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${ECDH_CONN_PSI_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${ECDH_CONN_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} ${TOOLS_TARGET} ${PROTOBUF_TARGET} TBB::tbb TCMalloc)
target_link_libraries(${ECDH_CONN_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} ${TOOLS_TARGET} ${PROTOBUF_TARGET} TBB::tbb)
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ add_library(${ECDH_MULTI_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${ECDH_MULTI_PSI_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${ECDH_MULTI_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} TBB::tbb TCMalloc)
target_link_libraries(${ECDH_MULTI_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} TBB::tbb)
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,8 @@ class CalculatorCache : public std::enable_shared_from_this<CalculatorCache>
for (auto const& it : m_plainData)
{
it->release();
// free after release
MallocExtension::instance()->ReleaseFreeMemory();
}
m_cipherRef.clear();
std::map<bcos::bytes, CipherRefDetail>().swap(m_cipherRef);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ void EcdhMultiPSIMaster::blindData()
});
// can release databatch after encrypted
dataBatch->release();
// free after release
MallocExtension::instance()->ReleaseFreeMemory();
ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: encrypt data success")
<< LOG_KV("dataSize", cipher.size()) << LOG_KV("task", m_taskID)
<< LOG_KV("seq", seq)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ void EcdhMultiPSIPartner::onReceiveRandomA(bcos::bytesPointer _randA)
<< LOG_KV("size", dataBatch->size())
<< printTaskInfo(m_taskState->task()) << LOG_KV("seq", seq);
dataBatch->release();
// free after release
MallocExtension::instance()->ReleaseFreeMemory();
} while (!m_taskState->sqlReader());
}
catch (std::exception& e)
Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/ecdh-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
file(GLOB_RECURSE SRCS *.cpp)
add_library(${ECDH_2PC_PSI_TARGET} ${SRCS})
target_link_libraries(${ECDH_2PC_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} ${TOOLS_TARGET} TCMalloc)
target_link_libraries(${ECDH_2PC_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TOOLS_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/labeled-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,4 @@ add_library(${LABELED_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${LABELED_PSI_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)

target_link_libraries(${LABELED_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} apsi SEAL::seal Kuku::kuku TBB::tbb TCMalloc)
target_link_libraries(${LABELED_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} apsi SEAL::seal Kuku::kuku TBB::tbb)
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ file(GLOB_RECURSE SRCS *.cpp)
add_library(${PSI_FRAMEWORK_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
target_include_directories(${PSI_FRAMEWORK_TARGET} PUBLIC
$<BUILD_INTERFACE:${TARS_HEADER_DIR}>)
target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} ${TARS_PROTOCOL_TARGET} TCMalloc)
target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC TCMalloc ${FRONT_TARGET} ${PROTOCOL_TARGET} ${TARS_PROTOCOL_TARGET} ${IO_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-computing/ppc-psi/src/ra2018-psi/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
file(GLOB_RECURSE SRCS *.cpp)
add_library(${RA2018_PSI_TARGET} ${SRCS})
target_link_libraries(${RA2018_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} ${TOOLS_TARGET})
target_link_libraries(${RA2018_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TOOLS_TARGET})
4 changes: 2 additions & 2 deletions cpp/wedpr-computing/ppc-psi/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ set(TEST_BINARY_NAME test-ppc-psi)
add_executable(${TEST_BINARY_NAME} ${SOURCES})
target_include_directories(${TEST_BINARY_NAME} PRIVATE .)

target_link_libraries(${TEST_BINARY_NAME} ${BS_ECDH_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
target_link_libraries(${TEST_BINARY_NAME} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
target_link_libraries(${TEST_BINARY_NAME} ${BS_ECDH_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
target_link_libraries(${TEST_BINARY_NAME} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
add_test(NAME test-ppc-psi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
4 changes: 2 additions & 2 deletions cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ find_package(OpenSSL REQUIRED)
message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}")
message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}")

target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB} TCMalloc)
target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB})

if (ENABLE_IPP_CRYPTO)
find_package(ipp-crypto REQUIRED)
target_link_libraries(${CRYPTO_TARGET} PUBLIC ipp-crypto::crypto_mb)
endif ()
endif ()
2 changes: 1 addition & 1 deletion cpp/wedpr-helper/libhelper/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})

add_library(${HELPER_TARGET} ${SRCS})
find_package(Boost COMPONENTS filesystem program_options)
target_link_libraries(${HELPER_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} Boost::filesystem Boost::program_options)
target_link_libraries(${HELPER_TARGET} PUBLIC Boost::filesystem Boost::program_options)
2 changes: 1 addition & 1 deletion cpp/wedpr-helper/ppc-tools/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ aux_source_directory(src/config SRCS)

add_library(${TOOLS_TARGET} ${SRCS})

target_link_libraries(${TOOLS_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${BOOST_SERIALIZATION} ${CPU_FEATURES_LIB})
target_link_libraries(${TOOLS_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BOOST_SERIALIZATION} ${CPU_FEATURES_LIB})

if (TESTS)
enable_testing()
Expand Down
20 changes: 13 additions & 7 deletions cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ using namespace ppc::protocol;
using namespace ppc::storage;
using namespace bcos;

void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt)
void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt, bool requireTransport)
{
// load the network config
PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config");
PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config")
<< LOG_KV("requireTransport", requireTransport);
// gateway default enable-ssl
loadNetworkConfig(
m_gatewayConfig.networkConfig, _pt, "gateway", NetworkConfig::DefaultRpcListenPort, false);
Expand Down Expand Up @@ -73,11 +74,16 @@ void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt)
InvalidConfig() << errinfo_comment(
"The value of gateway.seq_sync_period_ms must no little than 3s"));
}
// load the grpcConfig
m_grpcConfig = loadGrpcConfig("transport", _pt);
// load the GrpcServerConfig
loadEndpointConfig(
m_gatewayConfig.grpcServerConfig->mutableEndPoint(), false, "transport", _pt);
// only need load grpc config when requireTransport
if (requireTransport)
{
PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load grpc config");
// load the grpcConfig
m_grpcConfig = loadGrpcConfig("transport", _pt);
// load the GrpcServerConfig
loadEndpointConfig(
m_gatewayConfig.grpcServerConfig->mutableEndPoint(), false, "transport", _pt);
}
// the agencyID
m_agencyID = _pt.get<std::string>("agency.id", "");
if (m_agencyID.empty())
Expand Down
6 changes: 3 additions & 3 deletions cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,12 @@ class PPCConfig
loadRpcConfig(iniConfig);
}

void loadGatewayConfig(std::string const& _configPath)
void loadGatewayConfig(std::string const& _configPath, bool requireTransport)
{
PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig") << LOG_KV("path", _configPath);
boost::property_tree::ptree iniConfig;
boost::property_tree::read_ini(_configPath, iniConfig);
loadGatewayConfig(iniConfig);
loadGatewayConfig(iniConfig, requireTransport);
}

void loadFrontConfig(bool requireTransport,
Expand All @@ -180,7 +180,7 @@ class PPCConfig
loadNetworkConfig(m_rpcConfig, _pt, "rpc", NetworkConfig::DefaultRpcListenPort, true);
}

virtual void loadGatewayConfig(boost::property_tree::ptree const& _pt);
virtual void loadGatewayConfig(boost::property_tree::ptree const& _pt, bool requireTransport);


NetworkConfig const& rpcConfig() const { return m_rpcConfig; }
Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-main/air-node/AirNodeInitializer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ void AirNodeInitializer::initGateway(std::string const& _configPath)
INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath;
// not specify the certPath in air-mode
auto config = m_nodeInitializer->config();
config->loadGatewayConfig(_configPath);
config->loadGatewayConfig(_configPath, false);

auto threadPool = std::make_shared<bcos::ThreadPool>(
"gateway", config->gatewayConfig().networkConfig.threadPoolSize);
Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-main/air-node/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
aux_source_directory(. SRC_LIST)

add_executable(${AIR_BINARY_NAME} ${SRC_LIST})
target_link_libraries(${AIR_BINARY_NAME} PUBLIC ${GATEWAY_TARGET} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc)
target_link_libraries(${AIR_BINARY_NAME} PUBLIC ${GATEWAY_TARGET} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-main/gateway/GatewayInitializer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ void GatewayInitializer::init(std::string const& _configPath)
INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath;
auto config = std::make_shared<PPCConfig>();

config->loadGatewayConfig(_configPath);
config->loadGatewayConfig(_configPath, true);
auto threadPool = std::make_shared<bcos::ThreadPool>(
"gateway", config->gatewayConfig().networkConfig.threadPoolSize);

Expand Down
2 changes: 1 addition & 1 deletion cpp/wedpr-main/pro-node/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR})
aux_source_directory(./ SRC_LIST)
add_executable(${PRO_BINARY_NAME} ${SRC_LIST})

target_link_libraries(${PRO_BINARY_NAME} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc)
target_link_libraries(${PRO_BINARY_NAME} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET})
2 changes: 1 addition & 1 deletion cpp/wedpr-storage/ppc-io/src/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
file(GLOB_RECURSE SRCS *.cpp)
add_library(${IO_TARGET} ${SRCS})
# Note: the DataBatch depends on tbb
target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} TCMalloc ${HDFS_LIB} ${CPU_FEATURES_LIB})
target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${HDFS_LIB} ${CPU_FEATURES_LIB})
2 changes: 1 addition & 1 deletion cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ add_library(${STORAGE_TARGET} ${SRCS})
find_package(redis++ REQUIRED)
find_package(unofficial-mysql-connector-cpp REQUIRED)

target_link_libraries(${STORAGE_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} redis++::redis++_static TCMalloc unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB})
target_link_libraries(${STORAGE_TARGET} PUBLIC redis++::redis++_static unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB})

0 comments on commit acbdd04

Please sign in to comment.