diff --git a/.github/workflows/cpp_workflow.yml b/.github/workflows/cpp_workflow.yml index f923ef49..2e50590e 100644 --- a/.github/workflows/cpp_workflow.yml +++ b/.github/workflows/cpp_workflow.yml @@ -63,7 +63,7 @@ jobs: export GCC='gcc-10' export CXX='g++-10' bash cpp/tools/install_depends.sh -o ubuntu - mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON ../ + mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ make -j3 - name: Build for macos if: runner.os == 'macOS' diff --git a/cpp/ppc-framework/io/DataBatch.h b/cpp/ppc-framework/io/DataBatch.h index 46ace021..b0da8d2e 100644 --- a/cpp/ppc-framework/io/DataBatch.h +++ b/cpp/ppc-framework/io/DataBatch.h @@ -20,7 +20,6 @@ #pragma once #include "../Common.h" #include -#include #include #include #include @@ -199,18 +198,8 @@ class DataBatch DataSchema getDataSchema() const { return m_dataSchema; } - uint32_t dedup() - { - if (!m_data || m_data->empty()) - { - return 0; - } - tbb::parallel_sort(m_data->begin(), m_data->end()); - auto unique_end = std::unique(m_data->begin(), m_data->end()); - m_data->erase(unique_end, m_data->end()); - return m_data->size(); - } + std::shared_ptr>& mutableData() { return m_data; } private: std::shared_ptr> m_data; diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h index 237f1a65..ccfffdbe 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h @@ -21,9 +21,9 @@ #include "ppc-framework/Common.h" #include "ppc-framework/protocol/PPCMessageFace.h" - #include #include +#include namespace ppc::psi @@ -65,4 +65,17 @@ enum class CM2020PSIRetCode : int INVALID_TASK_PARAM = -3002 }; +inline uint32_t dedupDataBatch(ppc::io::DataBatch::Ptr dataBatch) +{ + if (!dataBatch || dataBatch->mutableData() == nullptr || dataBatch->mutableData()->empty()) + { + return 0; + } + auto& data = dataBatch->mutableData(); + tbb::parallel_sort(data->begin(), data->end()); + auto unique_end = std::unique(data->begin(), data->end()); + data->erase(unique_end, data->end()); + return data->size(); +} + } // namespace ppc::psi diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp index 2e4f46b0..0f4c92d1 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp @@ -908,7 +908,7 @@ void CM2020PSIReceiver::saveResults() } CM2020_PSI_LOG(INFO) << LOG_BADGE("before dedup") << LOG_KV("taskID", m_taskID) << LOG_KV("originCount", finalResults->size()); - m_resultCount = finalResults->dedup(); + m_resultCount = dedupDataBatch(finalResults); CM2020_PSI_LOG(INFO) << LOG_BADGE("after dedup") << LOG_KV("taskID", m_taskID) << LOG_KV("resultCount", m_resultCount); m_taskState->writeLines(finalResults, DataSchema::Bytes); diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp index 860f8bfd..d4018dc4 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp @@ -808,7 +808,7 @@ void CM2020PSISender::saveResults() } CM2020_PSI_LOG(INFO) << LOG_BADGE("before dedup") << LOG_KV("taskID", m_taskID) << LOG_KV("originCount", finalResults->size()); - m_resultCount = finalResults->dedup(); + m_resultCount = dedupDataBatch(finalResults); CM2020_PSI_LOG(INFO) << LOG_BADGE("after dedup") << LOG_KV("taskID", m_taskID) << LOG_KV("resultCount", m_resultCount); m_taskState->writeLines(finalResults, DataSchema::Bytes); diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h index ea2459ad..d024b335 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h @@ -2,6 +2,7 @@ #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h" #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h" #include "ppc-psi/src/psi-framework/TaskState.h" +#include namespace ppc::psi { diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h index 4a3076a6..30b57041 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h @@ -2,6 +2,7 @@ #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h" #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h" #include "ppc-psi/src/psi-framework/TaskState.h" +#include namespace ppc::psi { diff --git a/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt index 38465dbf..d26ddab5 100644 --- a/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt +++ b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt @@ -1,4 +1,4 @@ file(GLOB_RECURSE SRCS *.cpp) add_library(${IO_TARGET} ${SRCS}) # Note: the DataBatch depends on tbb -target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${HDFS_LIB} ${CPU_FEATURES_LIB} TBB::tbb) \ No newline at end of file +target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${HDFS_LIB} ${CPU_FEATURES_LIB}) \ No newline at end of file