Merge branch 'feature-milestone2' into feature-milestone2

Author: yanxinyi620

.github/workflows/cpp_workflow.yml

@@ -75,7 +75,6 @@ jobs:
         if: runner.os != 'Windows'
         run: |
           cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest
-          make coverage
 
   build_sdk:
     name: build sdk
@@ -129,20 +128,24 @@ jobs:
       - name: Build for macos
         if: runner.os == 'macOS'
         run: |
+          brew install lcov
           bash cpp/tools/install_depends.sh -o macos
-          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=OFF -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
       - name: Test
-        if: runner.os != 'Windows'
+        if: runner.os == 'macOS'
         run: |
           cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
-          make coverage
-      - uses: actions/upload-artifact@v2
+      - name: Test and upload coverage
+        if: runner.os == 'Linux'
+        run: |
+          cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
+      - uses: actions/upload-artifact@v4
         if: runner.os == 'macos'
         with:
           name: libppc-crypto-sdk-jni.dylib
           path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.dylib
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: runner.os == 'Windows'
         with:
           name: libppc-crypto-sdk-jni.dylib
@@ -185,7 +188,7 @@ jobs:
       - name: Prepare vcpkg
         if: runner.os != 'Windows'
         uses: friendlyanon/setup-vcpkg@v1
-        with: { committish: 7e3dcf74e37034eea358934a90a11d618520e139 }
+        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly-2022-07-28
@@ -206,19 +209,19 @@ jobs:
         run: |
           export OMP_NUM_THREADS=1
           cd build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: ppc-air-node-centos-x64
           path: ./cpp/build/bin/ppc-air-node
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: ppc-pro-node-centos-x64
           path: ./cpp/build/bin/ppc-pro-node
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: ppc-gateway-service-centos-x64
           path: ./cpp/build/bin/ppc-gateway-service
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: libppc-crypto-sdk-jni.so
-          path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so
+          path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so

.gitignore

@@ -1,3 +1,5 @@
+.venv
+
 # Prerequisites
 *.d
 
@@ -36,13 +38,15 @@ cmake-build-debug
 *.exe
 *.out
 *.app
+**/*.class
 
 build/**
 GPATH
 GRTAGS
 GTAGS
 deps
 deps/**
+**/bin
 
 d1-encrypted
 d2-encrypted
@@ -137,6 +141,7 @@ ENV/
 *.crt
 */bin
 */logs
+*/bin
 *.csv
 */*/logs
 */start.out

cpp/CMakeLists.txt

@@ -72,6 +72,12 @@ set(CEM_SOURCE "")
 if(BUILD_CEM)
     set(CEM_SOURCE "wedpr-computing/ppc-cem")
 endif()
+
+set(TRANSPORT_SDK_SOURCE_LIST 
+    wedpr-protocol 
+    wedpr-transport/ppc-front 
+    wedpr-transport/sdk wedpr-transport/sdk-wrapper)
+
 set(ALL_SOURCE_LIST 
         ${SDK_SOURCE_LIST} 
         wedpr-crypto/ppc-crypto 
@@ -83,24 +89,14 @@ set(ALL_SOURCE_LIST
         wedpr-computing/ppc-psi wedpr-computing/ppc-mpc wedpr-computing/ppc-pir ${CEM_SOURCE}
         wedpr-initializer wedpr-main)
 
-if(BUILD_WEDPR_TOOLKIT)
-    # fetch the python dependencies
-    option(FETCH_PYTHON_DEPS "Install python required modules if not available" ON)
-    message(STATUS "Python fetch dependencies: ${FETCH_PYTHON_DEPS}")
-    if(WIN32)
-        message(STATUS "Getting SWIG for Windows: ...")
-        include(swig)
-        message(STATUS "Getting SWIG for Windows: ...DONE")
-    endif()
-    add_subdirectory(wedpr-transport/sdk-wrapper)
-endif()
-
 if(BUILD_ALL)
     add_sources("${ALL_SOURCE_LIST}")
 elseif(BUILD_UDF)
     add_sources("${UDF_SOURCE_LIST}")
 elseif(BUILD_SDK)
     add_sources("${SDK_SOURCE_LIST}")
+elseif(BUILD_WEDPR_TOOLKIT)
+    add_sources("${TRANSPORT_SDK_SOURCE_LIST}")
 endif()
 ########### set the sources end ########### 
 

cpp/cmake/BuildInfo.cmake

@@ -1,6 +1,6 @@
 function(create_build_info)
     # Set build platform; to be written to BuildInfo.h
-    set(PPC_BUILD_OS "${CMAKE_SYSTEM_NAME}")
+    set(PPC_BUILD_OS "${CMAKE_HOST_SYSTEM_NAME}")
 
     if (CMAKE_COMPILER_IS_MINGW)
         set(PPC_BUILD_COMPILER "mingw")
@@ -27,7 +27,7 @@ function(create_build_info)
     # Generate header file containing useful build information
     add_custom_target(BuildInfo.h ALL
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-        COMMAND ${CMAKE_COMMAND} -DPPC_SOURCE_DIR="${PROJECT_SOURCE_DIR}"
+        COMMAND ${CMAKE_COMMAND} -DPPC_SOURCE_DIR="${PROJECT_SOURCE_DIR}/.."
         -DPPC_BUILDINFO_IN="${CMAKE_CURRENT_SOURCE_DIR}/cmake/templates/BuildInfo.h.in"
         -DPPC_DST_DIR="${PROJECT_BINARY_DIR}/include"
         -DPPC_CMAKE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/cmake"

cpp/cmake/CompilerSettings.cmake

@@ -101,7 +101,7 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
         add_compile_options(-fstack-protector)
         add_compile_options(-Winconsistent-missing-override)
         # Some Linux-specific Clang settings.  We don't want these for OS X.
-        if ("${CMAKE_SYSTEM_NAME}" MATCHES "Linux")
+        if ("${CMAKE_HOST_SYSTEM_NAME}" MATCHES "Linux")
             # Tell Boost that we're using Clang's libc++.   Not sure exactly why we need to do.
             add_definitions(-DBOOST_ASIO_HAS_CLANG_LIBCXX)
             # Use fancy colors in the compiler diagnostics

cpp/cmake/Coverage.cmake

@@ -26,12 +26,12 @@ function(config_coverage TARGET REMOVE_FILE_PATTERN)
         message(STATUS "coverage REMOVE_FILE_PATTERN: " ${REMOVE_FILE_PATTERN})
         if (APPLE)
             add_custom_target(${TARGET}
-                COMMAND ${LCOV_TOOL} -keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -i -d ${CMAKE_BINARY_DIR}/
+                COMMAND ${LCOV_TOOL} -keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -d ${CMAKE_BINARY_DIR}/
                 COMMAND ${LCOV_TOOL} -keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -r ${CMAKE_BINARY_DIR}/coverage.info.in '*MacOS*' '/usr*' '.*vcpkg_installed*' '.*boost/*' '*test*' '*build*' '*deps*' ${REMOVE_FILE_PATTERN} -o ${CMAKE_BINARY_DIR}/coverage.info
                 COMMAND genhtml --keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -q -o ${CMAKE_BINARY_DIR}/CodeCoverage ${CMAKE_BINARY_DIR}/coverage.info)
         else()
             add_custom_target(${TARGET}
-                COMMAND ${LCOV_TOOL} --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -i -d ${CMAKE_BINARY_DIR}/
+                COMMAND ${LCOV_TOOL} --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -d ${CMAKE_BINARY_DIR}/
                 COMMAND ${LCOV_TOOL} --rc lcov_branch_coverage=1 -r ${CMAKE_BINARY_DIR}/coverage.info.in '*MacOS*' '/usr*' '.*vcpkg_installed*' '.*boost/*' '*test*' '*build*' '*deps*' ${REMOVE_FILE_PATTERN} -o ${CMAKE_BINARY_DIR}/coverage.info
                 COMMAND genhtml --rc lcov_branch_coverage=1 -q -o ${CMAKE_BINARY_DIR}/CodeCoverage ${CMAKE_BINARY_DIR}/coverage.info)
         endif()

cpp/cmake/Dependencies.cmake

@@ -21,17 +21,19 @@ find_package(unofficial-sodium CONFIG REQUIRED)
 
 
 ##### the full-dependencies #####
-if(BUILD_ALL)
-    find_package(TBB REQUIRED)
+if(BUILD_ALL OR BUILD_WEDPR_TOOLKIT)
     find_package(jsoncpp REQUIRED)
-
+    find_package(TBB REQUIRED)
+    find_package(gRPC REQUIRED)
     find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED)
+endif()
+
+if(BUILD_ALL)
     # tcmalloc
     include(ProjectTCMalloc)
 
     find_package(SEAL REQUIRED)
     find_package(Kuku REQUIRED)
-    find_package(gRPC REQUIRED)
 
     # APSI: Note: APSI depends on seal 4.0 and Kuku 2.1
     include(ProjectAPSI)

cpp/cmake/Installlibhdfs3.cmake

@@ -1,6 +1,6 @@
 find_package(libxml2 REQUIRED)
 find_package(protobuf REQUIRED)
-if ("${CMAKE_SYSTEM_NAME}" MATCHES "Linux")
+if ("${CMAKE_HOST_SYSTEM_NAME}" MATCHES "Linux")
 	find_package(unofficial-libuuid REQUIRED)
 endif()
 find_package(libhdfs3 REQUIRED)

cpp/cmake/Options.cmake

@@ -82,9 +82,10 @@ macro(configure_project)
         if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*SSE4.2.*")
             set(ENABLE_SSE ON)
         endif ()
-    elseif(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+    elseif(NOT ${CMAKE_HOST_SYSTEM_NAME} MATCHES "Windows")
         # detect sse4_2
         FILE(READ "/proc/cpuinfo" SUPPORTED_INSTRUCTIONS)
+        message("* Linux SUPPORTED_INSTRUCTIONS: ${SUPPORTED_INSTRUCTIONS}")
         if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*sse4_2.*")
             set(ENABLE_SSE ON)
         endif ()
@@ -130,13 +131,23 @@ macro(configure_project)
         set(VISIBILITY_FLAG "")
         set(BUILD_ALL OFF)
     endif()
+    if (BUILD_WEDPR_TOOLKIT)
+        set(VISIBILITY_FLAG "")
+        set(BUILD_ALL OFF)
+    endif()
     if (BUILD_ALL)
         # install all dependencies
         list(APPEND VCPKG_MANIFEST_FEATURES "all")
+        if(ENABLE_SSE)
+            # enable sse for libhdfs3
+            list(APPEND VCPKG_MANIFEST_FEATURES "sse-libhdfs3")
+        else()
+            list(APPEND VCPKG_MANIFEST_FEATURES "default-libhdfs3")
+        endif()
     endif()
-    if(ENABLE_SSE)
-        # enable sse for libhdfs3
-        list(APPEND VCPKG_MANIFEST_FEATURES "sse")
+    if (BUILD_WEDPR_TOOLKIT)
+        # install wedpr dependencies
+        list(APPEND VCPKG_MANIFEST_FEATURES "toolkit")
     endif()
     # cpp_features
     if(ENABLE_CPU_FEATURES)
@@ -156,7 +167,7 @@ macro(print_config NAME)
     message("-- CMAKE_BUILD_TYPE    Build type                   ${CMAKE_BUILD_TYPE}")
     message("-- VCPKG_MANIFEST_FEATURES VCPKG manifest features  ${VCPKG_MANIFEST_FEATURES}")
     message("-- CMAKE_TOOLCHAIN_FILE Cmake toolchain file        ${CMAKE_TOOLCHAIN_FILE}")
-    message("-- TARGET_PLATFORM     Target platform              ${CMAKE_SYSTEM_NAME} ${ARCHITECTURE}")
+    message("-- TARGET_PLATFORM     Target platform              ${CMAKE_HOST_SYSTEM_NAME} ${ARCHITECTURE}")
     message("-- BUILD_STATIC        Build static                 ${BUILD_STATIC}")
     message("-- COVERAGE            Build code coverage          ${COVERAGE}")
     message("-- TESTS               Build tests                  ${TESTS}")
@@ -170,6 +181,7 @@ macro(print_config NAME)
     message("-- BUILD_SDK           BUILD SDK                    ${BUILD_SDK}")
     message("-- BUILD_UDF           BUILD UDF                    ${BUILD_UDF}")
     message("-- BUILD_WEDPR_TOOLKIT BUILD_WEDPR_TOOLKIT          ${BUILD_WEDPR_TOOLKIT}")
+    message("-- AUTO_GENERATE       AUTO_GENERATE                ${AUTO_GENERATE}")
     message("-- DEBUG                                            ${DEBUG}")
     message("------------------------------------------------------------------------")
     message("")

cpp/cmake/TargetSettings.cmake

@@ -138,7 +138,9 @@ set(WEDPR_TRANSPORT_SDK_TARGET wedpr-transport-sdk)
 
 # ==== the swig wrapper ===== 
 set(WEDPR_PYTHON_TRANSPORT "wedpr_python_transport")
-set(WEDPR_PYTHON_TRANSPORT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TRANSPORT})
+set(WEDPR_PYTHON_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python/generated/${WEDPR_PYTHON_TRANSPORT})
+set(WEDPR_PYTHON_TRANSPORT_LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/sdk-wrapper/python/bindings/libs)
 
-set(WEDPR_JAVA_TRANSPORT "wedpr_java_transport")
-set(WEDPR_JAVA_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/java/${WEDPR_JAVA_TRANSPORT})
+set(WEDPR_JAVA_TRANSPORT "wedpr_java_transport_jni")
+set(WEDPR_JAVA_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/java/generated/${WEDPR_JAVA_TRANSPORT})
+set(WEDPR_JAVA_TRANSPORT_LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/sdk-wrapper/java/bindings/src/main/resources/META-INF/native)

cpp/cmake/python.cmake

@@ -19,7 +19,13 @@ if(UNIX AND NOT APPLE)
 endif()
 
 # Find Python 3
-find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module)
+find_package(Python REQUIRED COMPONENTS Interpreter Development)
+message("Python_FOUND:${Python_FOUND}")
+message("Python_VERSION:${Python_VERSION}")
+message("Python_Development_FOUND:${Python_Development_FOUND}")
+message("Python_LIBRARIES:${Python_LIBRARIES}")
+message("Python_INCLUDE_DIRS:${Python_INCLUDE_DIRS}")
+
 list(APPEND CMAKE_SWIG_FLAGS "-py3" "-DPY3")
 
 function(search_python_module)

cpp/ppc-framework/Common.h

@@ -61,28 +61,6 @@ DERIVE_PPC_EXCEPTION(DataSchemaNotSetted);
 DERIVE_PPC_EXCEPTION(UnsupportedDataSchema);
 DERIVE_PPC_EXCEPTION(WeDPRException);
 
-constexpr static int MAX_PORT = 65535;
-constexpr static int DEFAULT_SECURITY_PARAM = 128;
-
-constexpr static size_t RSA_PUBLIC_KEY_PREFIX = 18;
-constexpr static size_t RSA_PUBLIC_KEY_TRUNC = 8;
-constexpr static size_t RSA_PUBLIC_KEY_TRUNC_LENGTH = 26;
-
-inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept
-{
-    if (p2pId.length() < RSA_PUBLIC_KEY_TRUNC_LENGTH)
-    {
-        return p2pId;
-    }
-    return p2pId.substr(RSA_PUBLIC_KEY_PREFIX, RSA_PUBLIC_KEY_TRUNC);
-}
-
-template <typename T>
-inline std::string printNodeID(T const& nodeID)
-{
-    return std::string(nodeID.begin(), nodeID.begin() + 8);
-}
-
 #if ENABLE_CPU_FEATURES
 #if X86
 static const cpu_features::X86Features CPU_FEATURES = cpu_features::GetX86Info().features;

cpp/ppc-framework/Helper.h

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file Common.h
+ * @author: yujiechen
+ * @date 2022-10-20
+ */
+#pragma once
+#include <sstream>
+#include <string>
+namespace ppc
+{
+constexpr static int MAX_PORT = 65535;
+constexpr static int DEFAULT_SECURITY_PARAM = 128;
+
+constexpr static size_t RSA_PUBLIC_KEY_PREFIX = 18;
+constexpr static size_t RSA_PUBLIC_KEY_TRUNC = 8;
+constexpr static size_t RSA_PUBLIC_KEY_TRUNC_LENGTH = 26;
+
+inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept
+{
+    if (p2pId.length() < RSA_PUBLIC_KEY_TRUNC_LENGTH)
+    {
+        return p2pId;
+    }
+    return p2pId.substr(RSA_PUBLIC_KEY_PREFIX, RSA_PUBLIC_KEY_TRUNC);
+}
+
+
+template <typename T>
+inline std::string_view printNodeID(T const& nodeID)
+{
+    size_t offset = nodeID.size() >= 15 ? 15 : nodeID.size();
+    return std::string_view((const char*)nodeID.data(), offset);
+}
+}  // namespace ppc

cpp/ppc-framework/front/FrontConfig.h

@@ -73,6 +73,8 @@ class FrontConfig
     virtual std::vector<std::string> const& getComponents() const { return m_components; }
     void setComponents(std::vector<std::string> const& components) { m_components = components; }
 
+    void addComponent(std::string const& component) { m_components.emplace_back(component); }
+
     std::vector<std::string>& mutableComponents() { return m_components; }
 
 protected: